{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "initial_id",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:42.961852Z",
     "start_time": "2025-01-27T13:52:36.560299Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:13.957111Z",
     "iopub.status.busy": "2025-01-27T15:33:13.956739Z",
     "iopub.status.idle": "2025-01-27T15:33:16.343619Z",
     "shell.execute_reply": "2025-01-27T15:33:16.342982Z",
     "shell.execute_reply.started": "2025-01-27T15:33:13.957075Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=10, micro=14, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.0\n",
      "torch 2.5.1+cu124\n",
      "cuda:0\n"
     ]
    }
   ],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "\n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd1ebc34a8798df6",
   "metadata": {},
   "source": [
    "# 数据加载与处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "939ac15db682d46d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.106860Z",
     "start_time": "2025-01-27T13:52:42.962847Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.345241Z",
     "iopub.status.busy": "2025-01-27T15:33:16.344810Z",
     "iopub.status.idle": "2025-01-27T15:33:16.382093Z",
     "shell.execute_reply": "2025-01-27T15:33:16.381316Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.345217Z"
    }
   },
   "outputs": [],
   "source": [
    "import unicodedata\n",
    "import re\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "\n",
    "# 因为西班牙语有一些是特殊字符，所以我们需要unicode转ascii，\n",
    "# 这样值变小了，因为unicode太大\n",
    "# 将 Unicode 字符串转换为 ASCII 字符串，去除其中的重音符号\n",
    "def unicode_to_ascii(s):\n",
    "    # NFD是转换方法，把每一个字节拆开，Mn是重音，所以去除\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "# #下面我们找个样本测试一下\n",
    "# # 加u代表对字符串进行unicode编码\n",
    "# en_sentence = u\"May I borrow this book?\"\n",
    "# sp_sentence = u\"¿Puedo tomar prestado este libro?\"\n",
    "# \n",
    "# print(unicode_to_ascii(en_sentence))\n",
    "# print(unicode_to_ascii(sp_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f631de2ef6279160",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.114681Z",
     "start_time": "2025-01-27T13:52:43.108854Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.383147Z",
     "iopub.status.busy": "2025-01-27T15:33:16.382756Z",
     "iopub.status.idle": "2025-01-27T15:33:16.387408Z",
     "shell.execute_reply": "2025-01-27T15:33:16.386803Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.383125Z"
    }
   },
   "outputs": [],
   "source": [
    "def preprocess_sentence(w):\n",
    "    # 变为小写，去掉多余的空格，变成小写，id少一些\n",
    "    w = unicode_to_ascii(w.lower().strip())\n",
    "\n",
    "    # 特定标点符号（?.!,¿）前后添加空格\n",
    "    # eg: \"he is a boy.\" => \"he is a boy . \"\n",
    "    # 这样可以使得模型更容易分辨单词和标点符号\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    # 因为可能有多余空格，替换为一个空格，所以处理一下\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    # 将字符串 w 中所有非字母字符（包括标点符号 ?.!,¿）替换为空格\n",
    "    # [^...] 表示匹配不在括号内的字符。\n",
    "    # a-zA-Z 匹配所有大小写字母。\n",
    "    # ?.!,¿ 匹配特定的标点符号。\n",
    "    #+ 表示匹配一个或多个连续字符。\n",
    "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "\n",
    "    #  先去除末尾空白字符，再去除首尾空白字符。\n",
    "    w = w.rstrip().strip()\n",
    "    return w\n",
    "\n",
    "# print(preprocess_sentence(en_sentence))\n",
    "# print(preprocess_sentence(sp_sentence))\n",
    "# print(preprocess_sentence(sp_sentence).encode('utf-8'))  #¿是占用两个字节的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c36ecfc4cba8609b",
   "metadata": {},
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c78b45975b61bd8f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.119627Z",
     "start_time": "2025-01-27T13:52:43.116677Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.388353Z",
     "iopub.status.busy": "2025-01-27T15:33:16.388016Z",
     "iopub.status.idle": "2025-01-27T15:33:16.390858Z",
     "shell.execute_reply": "2025-01-27T15:33:16.390372Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.388330Z"
    }
   },
   "outputs": [],
   "source": [
    "# #zip例子\n",
    "# a = [[1, 2], [4, 5], [7, 8]]\n",
    "# # *a 是 Python 的解包操作符，将 a 解包为 3 个子列表：\n",
    "# # zip 函数将解包后的子列表按位置组合\n",
    "# zipped = list(zip(*a))\n",
    "# print(zipped)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f0e1f6b9133f7cc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.126777Z",
     "start_time": "2025-01-27T13:52:43.122623Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.392877Z",
     "iopub.status.busy": "2025-01-27T15:33:16.392364Z",
     "iopub.status.idle": "2025-01-27T15:33:16.395301Z",
     "shell.execute_reply": "2025-01-27T15:33:16.394759Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.392838Z"
    }
   },
   "outputs": [],
   "source": [
    "# split_index1 = np.random.choice(a=[\"train\", \"test\"],\n",
    "#                                 replace=True, p=[0.9, 0.1], size=100)\n",
    "# print(len(split_index1))\n",
    "# print(\"-\" * 50)\n",
    "# split_index1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4bc7f9f4cb98ac0f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.582409Z",
     "start_time": "2025-01-27T13:52:43.128774Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.396220Z",
     "iopub.status.busy": "2025-01-27T15:33:16.395975Z",
     "iopub.status.idle": "2025-01-27T15:33:16.771440Z",
     "shell.execute_reply": "2025-01-27T15:33:16.770867Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.396200Z"
    }
   },
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "\n",
    "class LangPairDataset(Dataset):\n",
    "    fpath = Path(r\"./data_spa_en/spa.txt\")  #数据文件路径\n",
    "    cache_path = Path(r\"./.cache/lang_pair.npy\")  #缓存文件路径 \n",
    "    # 按照9:1划分训练集和测试集\n",
    "    split_index = np.random.choice(a=[\"train\", \"test\"], replace=True, p=[0.9, 0.1], size=118964)\n",
    "\n",
    "    def __init__(self, mode=\"train\", cache=False):\n",
    "        # if cache or not self.cache_path.exists() 表示：\n",
    "        # 如果启用缓存（cache 为 True），或者缓存文件不存在，则条件成立。\n",
    "        if cache or not self.cache_path.exists():\n",
    "            # parent 属性返回 self.cache_path 的父目录。\n",
    "            # mkdir 是 Path 对象的方法，用于创建目录。\n",
    "            # parents=True 表示递归创建父目录（如果父目录不存在）。\n",
    "            # exist_ok=True 表示如果目录已存在，不会抛出错误。\n",
    "            self.cache_path.parent.mkdir(exist_ok=True, parents=True)\n",
    "            with open(self.fpath, \"r\", encoding=\"utf-8\") as fd:\n",
    "                # 从文件中读取所有行，并将其存储在一个列表中\n",
    "                lines = fd.readlines()\n",
    "                # 将 lines 中的每一行按制表符 \\t 分割，并对每个分割后的部分调用 preprocess_sentence 函数进行预处理\n",
    "                lang_pair = [[preprocess_sentence(w) for w in l.split(\"\\t\")] for l in lines]\n",
    "                # 分离出目标语言和源语言\n",
    "                trg, src = zip(*lang_pair)\n",
    "                trg = np.array(trg)\n",
    "                src = np.array(src)\n",
    "                # 保存为npy文件,方便下次直接读取,不用再处理\n",
    "                np.save(self.cache_path, {\"trg\": trg, \"src\": src})\n",
    "        else:\n",
    "            # 从 .npy 文件中加载数据，并将其转换为 Python 字典（或其他对象）\n",
    "            # np.load 是 NumPy 提供的函数，用于从 .npy 或 .npz 文件中加载数据。\n",
    "            # allow_pickle=True 表示允许从 .npy 文件中加载 Python 对象。\n",
    "            # .item() 将其转换为 Python 对象（如字典、列表、字典等）\n",
    "            lang_pair = np.load(self.cache_path, allow_pickle=True).item()\n",
    "            trg = lang_pair[\"trg\"]\n",
    "            src = lang_pair[\"src\"]\n",
    "\n",
    "        # 按照index拿到训练集的 标签语言 --英语\n",
    "        self.trg = trg[self.split_index == mode]\n",
    "        # 按照index拿到训练集的源语言 --西班牙\n",
    "        self.src = src[self.split_index == mode]\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        return self.src[index], self.trg[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.src)\n",
    "\n",
    "\n",
    "train_ds = LangPairDataset(mode=\"train\")\n",
    "test_ds = LangPairDataset(mode=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "83a0eec9d0d9e91d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:43.587926Z",
     "start_time": "2025-01-27T13:52:43.583404Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.772462Z",
     "iopub.status.busy": "2025-01-27T15:33:16.772182Z",
     "iopub.status.idle": "2025-01-27T15:33:16.776100Z",
     "shell.execute_reply": "2025-01-27T15:33:16.775365Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.772441Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "source:si quieres sonar como un hablante nativo , debes estar dispuesto a practicar diciendo la misma frase una y otra vez de la misma manera en que un musico de banjo practica el mismo fraseo una y otra vez hasta que lo puedan tocar correctamente y en el tiempo esperado .\n",
      "target:if you want to sound like a native speaker , you must be willing to practice saying the same sentence over and over in the same way that banjo players practice the same phrase over and over until they can play it correctly and at the desired tempo .\n"
     ]
    }
   ],
   "source": [
    "print(\"source:{}\\ntarget:{}\".format(*train_ds[-1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "906e6d420362b515",
   "metadata": {},
   "source": [
    "## Tokenizer\n",
    "\n",
    "这里有两种处理方式，分别对应着 encoder 和 decoder 的 word embedding 是否共享，这里实现不共享的方案。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ae72149b871c746d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.281040Z",
     "start_time": "2025-01-27T13:52:43.589438Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:16.777040Z",
     "iopub.status.busy": "2025-01-27T15:33:16.776779Z",
     "iopub.status.idle": "2025-01-27T15:33:17.394669Z",
     "shell.execute_reply": "2025-01-27T15:33:17.394092Z",
     "shell.execute_reply.started": "2025-01-27T15:33:16.777017Z"
    }
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "\n",
    "def get_word_idx(ds, mode=\"src\", threshold=2):\n",
    "    #载入词表，看下词表长度，词表就像英语字典\n",
    "    word2idx = {\n",
    "        \"[PAD]\": 0,  # 填充 token\n",
    "        \"[BOS]\": 1,  # begin of sentence\n",
    "        \"[UNK]\": 2,  # 未知 token\n",
    "        \"[EOS]\": 3,  # end of sentence\n",
    "    }\n",
    "    idx2word = {value: key for key, value in word2idx.items()}\n",
    "    index = len(idx2word)\n",
    "    # 出现次数低于此的token舍弃\n",
    "    threshold = 1\n",
    "    # 如果数据集有很多个G，那是用for循环的，不能' '.join\n",
    "    word_list = \" \".join([pair[0 if mode == \"src\" else 1] for pair in ds]).split()\n",
    "    # print(type(word_list)) # <class 'list'>\n",
    "    # 统计词频,counter类似字典，key是单词，value是出现次数\n",
    "    counter = Counter(word_list)\n",
    "    # print(\"word count:\", len(counter))\n",
    "\n",
    "    for token, count in counter.items():\n",
    "        # 出现次数大于阈值的token加入词表\n",
    "        if count >= threshold:\n",
    "            word2idx[token] = index  # 加入词表\n",
    "            idx2word[index] = token  # 加入反向词典\n",
    "            index += 1\n",
    "\n",
    "    return word2idx, idx2word\n",
    "\n",
    "\n",
    "# 源语言词表  西班牙语\n",
    "src_word2idx, src_idx2word = get_word_idx(train_ds, mode=\"src\")\n",
    "# 目标语言词表 英语\n",
    "trg_word2idx, trg_idx2word = get_word_idx(train_ds, mode=\"trg\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "1585f8ee2b3174b7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.290344Z",
     "start_time": "2025-01-27T13:52:44.282033Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.395707Z",
     "iopub.status.busy": "2025-01-27T15:33:17.395419Z",
     "iopub.status.idle": "2025-01-27T15:33:17.405346Z",
     "shell.execute_reply": "2025-01-27T15:33:17.404716Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.395684Z"
    }
   },
   "outputs": [],
   "source": [
    "class Tokenizer:\n",
    "    def __init__(self, word2idx, idx2word, max_length=500,\n",
    "                 pad_idx=0, bos_idx=1, eos_idx=3, unk_idx=2):\n",
    "        self.word2idx = word2idx\n",
    "        self.idx2word = idx2word\n",
    "        self.max_length = max_length\n",
    "        self.pad_idx = pad_idx\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.unk_idx = unk_idx\n",
    "\n",
    "    def encode(self, text_list, padding_first=False, add_bos=True, add_eos=True,\n",
    "               return_mask=False):\n",
    "        # 如果padding_first == True，则padding加载前面，否则加载后面\n",
    "        # return_mask: 是否返回mask(掩码），mask用于指示哪些是padding的，哪些是真实的token \n",
    "        # 若启动bos_eos，则在句首和句尾分别添加bos和eos，则 add_bos=True, add_eos=True即都为1\n",
    "        max_length = min(self.max_length, add_bos + add_eos + max([len(text) for text in text_list]))\n",
    "        indices_list = []\n",
    "        for text in text_list:\n",
    "            # 如果词表中没有这个词，就用unk_idx代替，indices是一个list,里面是每个词的index,也就是一个样本的index\n",
    "            indices = [self.word2idx.get(word, self.unk_idx) for word in text[:max_length - add_eos - add_bos]]\n",
    "            if add_bos:\n",
    "                indices = [self.bos_idx] + indices\n",
    "            if add_eos:\n",
    "                indices = indices + [self.eos_idx]\n",
    "            if padding_first:  # padding加载前面，超参可以调\n",
    "                indices = [self.pad_idx] * (max_length - len(indices)) + indices\n",
    "            else:  # padding加载后面\n",
    "                indices = indices + [self.pad_idx] * (max_length - len(indices))\n",
    "            indices_list.append(indices)\n",
    "        input_ids = torch.tensor(indices_list)  # 转换为tensor\n",
    "        # mask是一个和input_ids一样大小的tensor，0代表token，1代表padding，mask用于去除padding的影响\n",
    "        masks = (input_ids == self.pad_idx).to(dtype=torch.float64)\n",
    "        return input_ids if not return_mask else (input_ids, masks)\n",
    "\n",
    "    def decode(self, indices_list, remove_bos=True, remove_eos=True, remove_pad=True, split=False):\n",
    "        text_list = []\n",
    "        for indices in indices_list:\n",
    "            text = []\n",
    "            for index in indices:\n",
    "                # 如果词表中没有这个词，就用unk_idx代替\n",
    "                word = self.idx2word.get(index, \"[UNK]\")\n",
    "                if remove_bos and word == \"[BOS]\":\n",
    "                    continue\n",
    "                if remove_eos and word == \"[EOS]\":  # 如果到达eos，就结束\n",
    "                    break\n",
    "                if remove_pad and word == \"[PAD]\":  # 如果到达pad，就结束\n",
    "                    break\n",
    "                text.append(word)  # 单词添加到列表中\n",
    "            # 把列表中的单词拼接，变为一个句子\n",
    "            text_list.append(\" \".join(text) if not split else text)\n",
    "        return text_list\n",
    "\n",
    "\n",
    "#两个相对于1个toknizer的好处是embedding的参数量减少\n",
    "# 源语言tokenizer\n",
    "src_tokenizer = Tokenizer(word2idx=src_word2idx, idx2word=src_idx2word)\n",
    "# 目标语言tokenizer\n",
    "trg_tokenizer = Tokenizer(word2idx=trg_word2idx, idx2word=trg_idx2word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "602e9cd5a872b5a5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.296339Z",
     "start_time": "2025-01-27T13:52:44.292339Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.406602Z",
     "iopub.status.busy": "2025-01-27T15:33:17.406130Z",
     "iopub.status.idle": "2025-01-27T15:33:17.409460Z",
     "shell.execute_reply": "2025-01-27T15:33:17.408880Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.406579Z"
    }
   },
   "outputs": [],
   "source": [
    "# # trg_tokenizer.encode([[\"hello\"], [\"hello\", \"world\"]], add_bos=True, add_eos=False,return_mask=True)\n",
    "# raw_text = [\"hello world\".split(), \"tokenize text datas with batch\".split(), \"this is a test\".split()]\n",
    "# indices,mask = trg_tokenizer.encode(raw_text, padding_first=False, add_bos=True, add_eos=True,return_mask=True)\n",
    "# \n",
    "# print(\"raw text\"+'-'*10)\n",
    "# for raw in raw_text:\n",
    "#     print(raw)\n",
    "# print(\"mask\"+'-'*10)\n",
    "# for m in mask:\n",
    "#     print(m)\n",
    "# print(\"indices\"+'-'*10)\n",
    "# for index in indices:\n",
    "#     print(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1a5102956b2ec3f1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.302231Z",
     "start_time": "2025-01-27T13:52:44.298340Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.410411Z",
     "iopub.status.busy": "2025-01-27T15:33:17.410168Z",
     "iopub.status.idle": "2025-01-27T15:33:17.412946Z",
     "shell.execute_reply": "2025-01-27T15:33:17.412392Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.410388Z"
    }
   },
   "outputs": [],
   "source": [
    "# decode_text = trg_tokenizer.decode(indices.tolist(), remove_bos=False, remove_eos=False, remove_pad=False)\n",
    "# print(\"decode text\"+'-'*10)\n",
    "# for decode in decode_text:\n",
    "#     print(decode)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df29f25d3c740e46",
   "metadata": {},
   "source": [
    "## DataLoader\n",
    "\n",
    "encoder设置mask的原因：在softmax计算时，padding的位置计算结果趋近于0\n",
    "\n",
    "decoder设置mask的原因：在计算loss时，padding的位置的loss不参与计算"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "caca3658e2a7a115",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.311318Z",
     "start_time": "2025-01-27T13:52:44.304224Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.413703Z",
     "iopub.status.busy": "2025-01-27T15:33:17.413527Z",
     "iopub.status.idle": "2025-01-27T15:33:17.418731Z",
     "shell.execute_reply": "2025-01-27T15:33:17.418217Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.413683Z"
    }
   },
   "outputs": [],
   "source": [
    "def collate_fn(batch):\n",
    "    # 取batch内第0列进行分词，赋给src_words\n",
    "    src_words = [pair[0].split() for pair in batch]\n",
    "    # 取batch内第1列进行分词，赋给trg_words\n",
    "    trg_words = [pair[1].split() for pair in batch]\n",
    "\n",
    "    # [PAD] [BOS] src [EOS]\n",
    "    encoder_inputs, encoder_inputs_mask = src_tokenizer.encode(src_words, padding_first=True, add_bos=True,\n",
    "                                                               add_eos=True, return_mask=True)\n",
    "\n",
    "    # 目标语言 [BOS] trg [PAD]\n",
    "    decoder_inputs = trg_tokenizer.encode(trg_words, padding_first=False, add_bos=True, add_eos=False,\n",
    "                                          return_mask=False)\n",
    "\n",
    "    # 用于后续计算loss\n",
    "    # trg [EOS] [PAD]\n",
    "    decoder_labels, decoder_labels_mask = trg_tokenizer.encode(trg_words, padding_first=False, add_bos=False,\n",
    "                                                               add_eos=True, return_mask=True)\n",
    "\n",
    "    return {\n",
    "        \"encoder_inputs\": encoder_inputs.to(device),\n",
    "        \"encoder_inputs_mask\": encoder_inputs_mask.to(device),\n",
    "        \"decoder_inputs\": decoder_inputs.to(device),\n",
    "        \"decoder_labels\": decoder_labels.to(device),\n",
    "        # mask用于去除padding的影响，计算loss时用\n",
    "        \"decoder_labels_mask\": decoder_labels_mask.to(device),\n",
    "    }  # 当返回的数据较多时，用dict返回比较合理\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "252c37d675f553a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.317518Z",
     "start_time": "2025-01-27T13:52:44.313306Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.419631Z",
     "iopub.status.busy": "2025-01-27T15:33:17.419399Z",
     "iopub.status.idle": "2025-01-27T15:33:17.422131Z",
     "shell.execute_reply": "2025-01-27T15:33:17.421626Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.419611Z"
    }
   },
   "outputs": [],
   "source": [
    "# for batch in train_loader:\n",
    "#     for key, value in batch.items():\n",
    "#         print(key)\n",
    "#         print(value.shape)\n",
    "#         print(\"-\"*50)\n",
    "#     break    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "929678e26f67eccc",
   "metadata": {},
   "source": [
    "# 定义模型"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19fb8fb3385416ab",
   "metadata": {},
   "source": [
    "## Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "998ca06a810799f6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.327992Z",
     "start_time": "2025-01-27T13:52:44.321515Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.425008Z",
     "iopub.status.busy": "2025-01-27T15:33:17.424749Z",
     "iopub.status.idle": "2025-01-27T15:33:17.429317Z",
     "shell.execute_reply": "2025-01-27T15:33:17.428784Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.424985Z"
    }
   },
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024, num_layers=1):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "\n",
    "    def forward(self, encoder_inputs):\n",
    "        # encoder_inputs.shape = [batch size, sequence_length]\n",
    "        embeds = self.embedding(encoder_inputs)\n",
    "        # embeds.shape = [batch size, sequence_length, embedding_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, sequence_length, hidden_dim]\n",
    "        # hidden.shape = [num_layers, batch size, hidden_dim]\n",
    "        return seq_output, hidden\n",
    "        # encoder_outputs[:,-1,:]==hidden[-1,:,:]  全是True 说明输出和hidden是相同的"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "4862d8fe85734897",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.334614Z",
     "start_time": "2025-01-27T13:52:44.329989Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.430316Z",
     "iopub.status.busy": "2025-01-27T15:33:17.430043Z",
     "iopub.status.idle": "2025-01-27T15:33:17.433292Z",
     "shell.execute_reply": "2025-01-27T15:33:17.432628Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.430294Z"
    }
   },
   "outputs": [],
   "source": [
    "# #把上面的Encoder写一个例子，看看输出的shape\n",
    "# encoder = Encoder(vocab_size=100, embedding_dim=256, hidden_dim=1024, num_layers=4)\n",
    "# # 生成一个形状为 (2, 50) 的随机整数张量，其中每个元素的取值范围是 [0, 100)\n",
    "# encoder_inputs = torch.randint(0, 100, (2, 50)) # [2,50]\n",
    "# encoder_outputs, hidden = encoder(encoder_inputs)\n",
    "# print(encoder_outputs.shape) # [2,50,1024]\n",
    "# print(hidden.shape) # [4,2,1024]\n",
    "# print(encoder_outputs[:,-1,:]) # 取最后一个时间步的输出\n",
    "# print(hidden[-1,:,:]) #取最后一层的hidden\n",
    "# print(encoder_outputs[:,-1,:]==hidden[-1,:,:]) # 全是True 说明输出和hidden是相同的"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "96f2a739975ee635",
   "metadata": {},
   "source": [
    "## BahdanauAttention公式\n",
    "score = FC(tanh(FC(EO) + FC(H))) 其中FC(EO)的FC是Wk,FC(H)的FC是Wq,最外面的FC是V \n",
    "\n",
    "attention_weights = softmax(score, axis = 1)  \n",
    "\n",
    "context = sum(attention_weights * EO, axis = 1) #对EO做加权求和，得到上下文向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "df0075e30e05dd4a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.344148Z",
     "start_time": "2025-01-27T13:52:44.336609Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.434386Z",
     "iopub.status.busy": "2025-01-27T15:33:17.433942Z",
     "iopub.status.idle": "2025-01-27T15:33:17.440479Z",
     "shell.execute_reply": "2025-01-27T15:33:17.439859Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.434364Z"
    }
   },
   "outputs": [],
   "source": [
    "class BahdanauAttention(nn.Module):\n",
    "    def __init__(self, hidden_dim=1024):\n",
    "        super().__init__()\n",
    "        # 对keys做运算，encoder的输出EO\n",
    "        self.Wk = nn.Linear(hidden_dim, hidden_dim)\n",
    "        # 对query做运算，decoder的隐藏状态\n",
    "        self.Wq = nn.Linear(hidden_dim, hidden_dim)\n",
    "        self.V = nn.Linear(hidden_dim, 1)\n",
    "\n",
    "    def forward(self, query, keys, values, attn_mask=None):\n",
    "        # query: hidden state，是decoder的隐藏状态，shape = [batch size,hidden_dim]\n",
    "        # keys: EO [batch size, sequence length, hidden_dim]\n",
    "        # values: EO  [batch size, sequence length, hidden_dim]\n",
    "        # attn_mask:[batch size, sequence length],这里是encoder_inputs_mask\n",
    "\n",
    "        # query.shape = [batch size, hidden_dim] -->通过unsqueeze(-2)增加维度 [batch size, 1, hidden_dim]\n",
    "        # keys.shape = [batch size, sequence length, hidden_dim]\n",
    "        # values.shape = [batch size, sequence length, hidden_dim]\n",
    "        score = self.V(F.tanh(self.Wk(keys) + self.Wq(query.unsqueeze(-2))))\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "\n",
    "        #这个mask是encoder_inputs_mask，用来mask掉padding的部分,让padding部分socres为0\n",
    "        if attn_mask is not None:\n",
    "            # 在最后增加一个维度，\n",
    "            # [batch size, sequence length] --> [batch size, sequence length, 1]\n",
    "            attn_mask = (attn_mask.unsqueeze(-1)) * -1e16  # 0还是0，1变为-1e16\n",
    "            # 即填充Pad的区域为-1e16，softmax后，这些位置的权重为0，不会影响输出\n",
    "            score += attn_mask\n",
    "        score = F.softmax(score, dim=-2)  # 对每一个词的score做softmax\n",
    "        # score.shape = [batch size, sequence length, 1]\n",
    "        #对每一个词的score和对应的value做乘法，然后在seq_len维度上求和，得到context_vector\n",
    "        context_vector = torch.mul(score, values).sum(dim=-2)\n",
    "        # context_vector.shape = [batch size, hidden_dim]\n",
    "        return context_vector, score\n",
    "        # score: 注意力权重，用于画图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "6f6fc98dab908ecf",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.350311Z",
     "start_time": "2025-01-27T13:52:44.346143Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.441458Z",
     "iopub.status.busy": "2025-01-27T15:33:17.441202Z",
     "iopub.status.idle": "2025-01-27T15:33:17.443987Z",
     "shell.execute_reply": "2025-01-27T15:33:17.443468Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.441438Z"
    }
   },
   "outputs": [],
   "source": [
    "# #tensor矩阵相乘\n",
    "# a = torch.randn(2, 3)\n",
    "# b = torch.randn(2, 3)\n",
    "# c = torch.mul(a, b) #增加维度\n",
    "# print(c.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "bc236d1b2f0f089a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.355408Z",
     "start_time": "2025-01-27T13:52:44.351307Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.445366Z",
     "iopub.status.busy": "2025-01-27T15:33:17.444833Z",
     "iopub.status.idle": "2025-01-27T15:33:17.448444Z",
     "shell.execute_reply": "2025-01-27T15:33:17.447785Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.445333Z"
    }
   },
   "outputs": [],
   "source": [
    "# #把上面的BahdanauAttention写一个例子，看看输出的shape\n",
    "# attention = BahdanauAttention(hidden_dim=1024)\n",
    "# query = torch.randn(2, 1024) #Decoder的隐藏状态\n",
    "# keys = torch.randn(2, 50, 1024) #EO\n",
    "# values = torch.randn(2, 50, 1024) #EO\n",
    "# attn_mask = torch.randint(0, 2, (2, 50))\n",
    "# context_vector, scores = attention(query, keys, values, attn_mask)\n",
    "# print(context_vector.shape)\n",
    "# print(scores.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1517bf2e93e55643",
   "metadata": {},
   "source": [
    "## Decoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "731b9a63e710aaad",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.363969Z",
     "start_time": "2025-01-27T13:52:44.356399Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.449592Z",
     "iopub.status.busy": "2025-01-27T15:33:17.449276Z",
     "iopub.status.idle": "2025-01-27T15:33:17.457189Z",
     "shell.execute_reply": "2025-01-27T15:33:17.456439Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.449562Z"
    }
   },
   "outputs": [],
   "source": [
    "class Decoder(nn.Module):\n",
    "    def __init__(self, vocab_size, embedding_dim=256, hidden_dim=1024, num_layers=1):\n",
    "        super().__init__()\n",
    "        self.embedding = nn.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = nn.GRU(embedding_dim + hidden_dim, hidden_dim, num_layers=num_layers, batch_first=True)\n",
    "        # 最后分类,词典大小是多少，就输出多少个分类\n",
    "        self.fc = nn.Linear(hidden_dim, vocab_size)\n",
    "        # 定义一个 Dropout 层，随机丢弃 60% 的神经元输出\n",
    "        self.dropout = nn.Dropout(0.6)\n",
    "        # 定义一个 BahdanauAttention 层,得到的context_vector\n",
    "        self.attention = BahdanauAttention(hidden_dim=hidden_dim)\n",
    "\n",
    "    def forward(self, decoder_inputs, hidden, encoder_outputs, attn_mask=None):\n",
    "        # attn_mask是encoder_inputs_mask,用来mask掉padding的部分,让padding部分socres为0\n",
    "\n",
    "        # decoder_inputs.shape = [batch size, 1]\n",
    "        assert len(decoder_inputs.shape) == 2 and decoder_inputs.shape[\n",
    "            -1] == 1, f\"decoder_input.shape = {decoder_inputs.shape} is not valid\"\n",
    "\n",
    "        # hidden.shape = [batch size, hidden_dim]，decoder_hidden,\n",
    "        # 而第一次使用的是encoder的hidden\n",
    "        assert len(hidden.shape) == 2, f\"hidden.shape = {hidden.shape} is not valid\"\n",
    "\n",
    "        # encoder_outputs.shape = [batch size, sequence length, hidden_dim]\n",
    "        assert len(encoder_outputs.shape) == 3, f\"encoder_outputs.shape = {encoder_outputs.shape} is not valid\"\n",
    "\n",
    "        # context_vector.shape = [batch_size, hidden_dim]\n",
    "        context_vector, attention_score = self.attention(query=hidden, keys=encoder_outputs, values=encoder_outputs,\n",
    "                                                         attn_mask=attn_mask)\n",
    "\n",
    "        # decoder_input.shape = [batch size, 1]\n",
    "        embeds = self.embedding(decoder_inputs)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim]\n",
    "        # context_vector.shape = [batch size, hidden_dim] -->unsqueeze(-2)增加维度 [batch size, 1, hidden_dim]\n",
    "        embeds = torch.cat([context_vector.unsqueeze(-2), embeds], dim=-1)\n",
    "        # embeds.shape = [batch size, 1, embedding_dim+hidden_dim]\n",
    "        seq_output, hidden = self.gru(embeds)\n",
    "        # seq_output.shape = [batch size, 1, hidden_dim]\n",
    "        logits = self.fc(seq_output)\n",
    "        # logits.shape = [batch size, 1, vocab_size]\n",
    "        # attention_score.shape = [batch size, sequence length, 1]\n",
    "        return logits, hidden, attention_score"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70d7459e2ecbd5de",
   "metadata": {},
   "source": [
    "## Seq2Seq模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "514f7b41823ae373",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:44.375072Z",
     "start_time": "2025-01-27T13:52:44.364965Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.458661Z",
     "iopub.status.busy": "2025-01-27T15:33:17.458354Z",
     "iopub.status.idle": "2025-01-27T15:33:17.469663Z",
     "shell.execute_reply": "2025-01-27T15:33:17.469039Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.458632Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Sequence2Sequence(nn.Module):\n",
    "    def __init__(\n",
    "            self,\n",
    "            src_vocab_size,  # 输入词典大小\n",
    "            trg_vocab_size,  # 输出词典大小\n",
    "            encoder_embedding_dim=256,\n",
    "            #encoder_hidden_dim和decoder_hidden_dim必须相同，是因为BahdanauAttention设计的\n",
    "            encoder_hidden_dim=1024,\n",
    "            encoder_num_layers=1,\n",
    "            decoder_embedding_dim=256,\n",
    "            decoder_hidden_dim=1024,\n",
    "            decoder_num_layers=1,\n",
    "            bos_idx=1,\n",
    "            eos_idx=3,\n",
    "            max_length=512,\n",
    "    ):\n",
    "        super().__init__()\n",
    "        self.bos_idx = bos_idx\n",
    "        self.eos_idx = eos_idx\n",
    "        self.max_length = max_length\n",
    "        self.encoder = Encoder(src_vocab_size, embedding_dim=encoder_embedding_dim, hidden_dim=encoder_hidden_dim,\n",
    "                               num_layers=encoder_num_layers)\n",
    "        self.decoder = Decoder(trg_vocab_size, embedding_dim=decoder_embedding_dim, hidden_dim=decoder_hidden_dim,\n",
    "                               num_layers=decoder_num_layers)\n",
    "\n",
    "    def forward(self, encoder_inputs, decoder_inputs, attn_mask=None):\n",
    "        \"\"\"\n",
    "        用于训练\n",
    "        (1)根据encoder_inputs计算出eneoder_ouyputs和hidden，用与计算上下文context_vector和scores\n",
    "        (2)然后根据decoder_inputs,context_vector和hidden计算出decoder_outputs和新的hidden，decoder_outputs能算出预测的logits\n",
    "        (3)最后将logits拼接、score拼接，作为模型的输出\n",
    "        注意：训练是一个元素一个元素计算，decoder_inputs其实就是训练样本的正确结果，即logit是由样本提问和回答结果计算得到的，所以训练时，decoder_inputs是正确答案，而非模型预测的结果。\n",
    "        \"\"\"\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "        # decoding\n",
    "        batch_size, seq_len = decoder_inputs.shape\n",
    "        logits_list = []\n",
    "        scores_list = []\n",
    "        for i in range(seq_len):  # 串行训练\n",
    "            # 每次迭代生成一个时间步的预测，存储在 logits_list 中，并且记录注意力分数（如果有的话）在 scores_list 中，最后将预测的logits和注意力分数拼接并返回。\n",
    "            logits, hidden, scores = self.decoder(\n",
    "                decoder_inputs[:, i:i + 1],  # 取第i个时间步的输入作为decoder的输入\n",
    "                # 取最后一层的hidden，第一个时间步时，hidden是encoder的hidden，第二个时间步时，hidden是decoder的hidden\n",
    "                hidden[-1],\n",
    "                encoder_outputs,\n",
    "                attn_mask=attn_mask,\n",
    "            )\n",
    "            # logits.shape = [batch size, 1,vocab_size]\n",
    "            # scores.shape = [batch size, sequence length, 1]\n",
    "            logits_list.append(logits)  # 记录预测的logits，用于计算损失\n",
    "            scores_list.append(scores)  # 记录注意力分数，用于画图\n",
    "        # [batch size, seq_len, vocab_size], [batch size, seq_len, seq_length]\n",
    "        return torch.cat(logits_list, dim=-2), torch.cat(scores_list, dim=-1)\n",
    "\n",
    "    @torch.no_grad()  # 装饰器，禁止梯度计算\n",
    "    def infer(self, encoder_inputs, attn_mask=None):\n",
    "        \"\"\"\n",
    "        用于预测，\n",
    "        此时，decoder_inputs是开始标记，\n",
    "        模型根据encoder_inputs(即提问)计算出eceoder_ouyputs和hidden，\n",
    "        用与计算上下文context_vector和scores，\n",
    "        然后根据context_vector和hidden生成第一个词，\n",
    "        作为下一个decoder_inputs，用于生成第二个词，作为新的decoder_inputs，以此类推，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        \"\"\"\n",
    "        # infer用于预测\n",
    "\n",
    "        # encoder_input.shape = [1, sequence length],这只支持batch_size=1\n",
    "        # encoding\n",
    "        encoder_outputs, hidden = self.encoder(encoder_inputs)\n",
    "\n",
    "        # decoding\n",
    "        # 创建一个形状为 (1, 1) 的张量，并将其数据类型转换为 torch.int64\n",
    "        # shape为[1,1]，内容为开始标记\n",
    "        decoder_inputs = torch.Tensor([self.bos_idx]).reshape(1, 1).to(dtype=torch.int64)\n",
    "        decoder_pred = None\n",
    "        pred_list = []  # 预测序列\n",
    "        scores_list = []\n",
    "        # 从开始标记 bos_idx 开始，迭代地生成序列，直到生成结束标记 eos_idx 或达到最大长度 max_length。\n",
    "        for _ in range(self.max_length):\n",
    "            logits, hidden, scores = self.decoder(\n",
    "                decoder_inputs,\n",
    "                hidden[-1],\n",
    "                encoder_outputs,\n",
    "                attn_mask=attn_mask,\n",
    "            )\n",
    "            # logits.shape = [1, 1,vocab_size]\n",
    "            decoder_pred = logits.argmax(dim=-1)\n",
    "            # decoder_pred.shape = [1, 1]\n",
    "            decoder_inputs = decoder_pred\n",
    "            # reshape(-1):从(1,1)变为（1）标量\n",
    "            pred_list.append(decoder_pred.reshape(-1).item())  # 记录预测的词\n",
    "            # scores.shape = [1, sequence length, 1]\n",
    "            scores_list.append(scores)  # 记录注意力分数，用于画图\n",
    "\n",
    "            # 如果生成结束标记，则停止迭代\n",
    "            if decoder_pred == self.eos_idx:\n",
    "                break\n",
    "\n",
    "        # cat(scores_list,dim=-1):[1, sequence length, sequence length]\n",
    "        return pred_list, torch.cat(scores_list, dim=-1)  # 返回预测序列和注意力分数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d119467be4a8eb18",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.185268Z",
     "start_time": "2025-01-27T13:52:44.376068Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.470595Z",
     "iopub.status.busy": "2025-01-27T15:33:17.470370Z",
     "iopub.status.idle": "2025-01-27T15:33:17.923698Z",
     "shell.execute_reply": "2025-01-27T15:33:17.923115Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.470573Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 50, 12474])\n",
      "torch.Size([2, 50, 50])\n"
     ]
    }
   ],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx))\n",
    "#做model的前向传播，看看输出的shape\n",
    "encoder_inputs = torch.randint(0, 100, (2, 50))\n",
    "decoder_inputs = torch.randint(0, 100, (2, 50))\n",
    "attn_mask = torch.randint(0, 2, (2, 50))\n",
    "logits, scores = model(encoder_inputs=encoder_inputs, decoder_inputs=decoder_inputs, attn_mask=attn_mask)\n",
    "print(logits.shape)\n",
    "print(scores.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "db1b6485db0bd5b9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.191754Z",
     "start_time": "2025-01-27T13:52:45.186260Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.924618Z",
     "iopub.status.busy": "2025-01-27T15:33:17.924362Z",
     "iopub.status.idle": "2025-01-27T15:33:17.928716Z",
     "shell.execute_reply": "2025-01-27T15:33:17.928070Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.924596Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 35,187,643 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "#帮我计算一下model的总参数量\n",
    "def count_parameters(model):\n",
    "    return sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "\n",
    "\n",
    "print(f\"The model has {count_parameters(model):,} trainable parameters\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d025b8a73c1b5776",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.612387Z",
     "start_time": "2025-01-27T13:52:45.192749Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:17.929757Z",
     "iopub.status.busy": "2025-01-27T15:33:17.929479Z",
     "iopub.status.idle": "2025-01-27T15:33:18.458326Z",
     "shell.execute_reply": "2025-01-27T15:33:18.457696Z",
     "shell.execute_reply.started": "2025-01-27T15:33:17.929734Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The model has 72,973,243 trainable parameters\n"
     ]
    }
   ],
   "source": [
    "#帮我初始化一个4层的GRU的model\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx), encoder_num_layers=4,\n",
    "                          decoder_num_layers=4)\n",
    "print(f\"The model has {count_parameters(model):,} trainable parameters\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39a17b7e22ff2c1d",
   "metadata": {},
   "source": [
    "# 训练"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a633e80b6643c86",
   "metadata": {},
   "source": [
    "## 损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fff18a09726ce0bb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.619581Z",
     "start_time": "2025-01-27T13:52:45.613388Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:18.459292Z",
     "iopub.status.busy": "2025-01-27T15:33:18.459055Z",
     "iopub.status.idle": "2025-01-27T15:33:18.464268Z",
     "shell.execute_reply": "2025-01-27T15:33:18.463738Z",
     "shell.execute_reply.started": "2025-01-27T15:33:18.459271Z"
    }
   },
   "outputs": [],
   "source": [
    "# 用来算一批数据的损失\n",
    "def cross_entropy_with_paddings(logits, labels, padding_mask=None):\n",
    "    # logits.shape = [batch size, sequence length, num of classes]\n",
    "    # labels.shape = [batch size, sequence length]\n",
    "    # padding_mask.shape = [batch size, sequence length]\n",
    "\n",
    "    batch_size, seq_len, num_classes = logits.shape\n",
    "    # F.cross_entropy 是 PyTorch 提供的函数，用于计算交叉熵损失\n",
    "    loss = F.cross_entropy(logits.reshape(batch_size * seq_len, num_classes), labels.reshape(-1),\n",
    "                           reduction='none')  # reduction='none'表示不对batch求平均\n",
    "    # loss.shape = [batch size * sequence length]\n",
    "    if padding_mask is None:  # 如果没有padding_mask，就直接求平均\n",
    "        loss = loss.mean()\n",
    "    else:\n",
    "        # 如果提供了 padding_mask，则将padding填充部分的损失去除后计算有效损失的均值。\n",
    "        # 首先，通过将 padding_mask reshape 成一维张量，并取 1 减去得到填充掩码。\n",
    "        # 这样填充部分的掩码值变为 1，非填充部分变为 0。\n",
    "        # 将损失张量与填充掩码相乘，这样填充部分的损失就会变为 0。\n",
    "        # 然后，计算非填充部分的损失和（sum）以及非填充部分的掩码数量（sum）作为有效损失的均值计算。\n",
    "        # (因为上面我们设计的mask的token是0，所以这里是1-padding_mask)\n",
    "\n",
    "        # 将padding_mask reshape成一维张量，mask部分为0，非mask部分为1\n",
    "        padding_mask = 1 - padding_mask.reshape(-1)\n",
    "        loss = torch.mul(loss, padding_mask).sum() / padding_mask.sum()\n",
    "\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "23808a399cd7ed2c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.626307Z",
     "start_time": "2025-01-27T13:52:45.620574Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:18.465094Z",
     "iopub.status.busy": "2025-01-27T15:33:18.464917Z",
     "iopub.status.idle": "2025-01-27T15:33:18.470864Z",
     "shell.execute_reply": "2025-01-27T15:33:18.470181Z",
     "shell.execute_reply.started": "2025-01-27T15:33:18.465074Z"
    }
   },
   "outputs": [],
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        self.save_dir = save_dir  # 保存路径\n",
    "        self.save_step = save_step  # 保存步数\n",
    "        self.save_best_only = save_best_only  # 是否只保存最好的模型\n",
    "        self.best_metric = -np.inf  # 最好的指标，指标不可能为负数，所以初始化为-1\n",
    "        # 创建保存路径\n",
    "        if not os.path.exists(self.save_dir):  # 如果不存在保存路径，则创建\n",
    "            os.makedirs(self.save_dir)\n",
    "\n",
    "    # 对象被调用时：当你将对象像函数一样调用时，Python 会自动调用 __call__ 方法。\n",
    "    # state_dict() 返回模型参数的字典，包括模型参数和优化器参数\n",
    "    # metric 是指标，可以是验证集的准确率，也可以是其他指标\n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return  # 不是保存步数，则直接返回\n",
    "\n",
    "        if self.save_best_only:\n",
    "            assert metric is not None  # 必须传入metric\n",
    "            if metric >= self.best_metric:  # 如果当前指标大于最好的指标\n",
    "                # save checkpoint\n",
    "                # 保存最好的模型，覆盖之前的模型，不保存step，只保存state_dict，即模型参数，不保存优化器参数\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"seq2seq_layer_2.ckpt\"))\n",
    "                self.best_metric = metric  # 更新最好的指标\n",
    "        else:\n",
    "            # 保存模型\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "            # 保存每个step的模型，不覆盖之前的模型，保存step，保存state_dict，即模型参数，不保存优化器参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "1ac803ec5a65f989",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.634181Z",
     "start_time": "2025-01-27T13:52:45.628299Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:18.472345Z",
     "iopub.status.busy": "2025-01-27T15:33:18.471809Z",
     "iopub.status.idle": "2025-01-27T15:33:18.477106Z",
     "shell.execute_reply": "2025-01-27T15:33:18.476450Z",
     "shell.execute_reply.started": "2025-01-27T15:33:18.472312Z"
    }
   },
   "outputs": [],
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        self.patience = patience  # 多少个step没有提升就停止训练\n",
    "        self.min_delta = min_delta  # 最小的提升幅度\n",
    "        self.best_metric = -np.inf  # 记录的最好的指标\n",
    "        self.counter = 0  # 计数器，记录连续多少个step没有提升\n",
    "\n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:  # 如果指标提升了\n",
    "            self.best_metric = metric  # 更新最好的指标\n",
    "            self.counter = 0  # 计数器清零\n",
    "        else:\n",
    "            self.counter += 1  # 计数器加一\n",
    "\n",
    "    @property  # 使用@property装饰器，使得 对象.early_stop可以调用，不需要()\n",
    "    def early_stop(self):\n",
    "        # 如果计数器大于等于patience，则返回True，停止训练\n",
    "        return self.counter >= self.patience"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8403c3b46f8ed34",
   "metadata": {},
   "source": [
    "## training & valuating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "d782c394b39d9d03",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.640534Z",
     "start_time": "2025-01-27T13:52:45.636175Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:18.478232Z",
     "iopub.status.busy": "2025-01-27T15:33:18.477841Z",
     "iopub.status.idle": "2025-01-27T15:33:18.482361Z",
     "shell.execute_reply": "2025-01-27T15:33:18.481809Z",
     "shell.execute_reply.started": "2025-01-27T15:33:18.478211Z"
    }
   },
   "outputs": [],
   "source": [
    "# 用来算整个test_loader的损失\n",
    "@torch.no_grad()  # 装饰器，禁止梯度计算\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    for batch in dataloader:\n",
    "        encoder_inputs = batch[\"encoder_inputs\"]\n",
    "        encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "        decoder_inputs = batch[\"decoder_inputs\"]\n",
    "        decoder_labels = batch[\"decoder_labels\"]\n",
    "        decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "        # 前向计算\n",
    "        logits, scores = model(\n",
    "            encoder_inputs=encoder_inputs,\n",
    "            decoder_inputs=decoder_inputs,\n",
    "            attn_mask=encoder_inputs_mask\n",
    "        )  # model就是seq2seq模型\n",
    "\n",
    "        # 验证集损失\n",
    "        loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "        loss_list.append(loss.cpu().item())\n",
    "\n",
    "    return np.mean(loss_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f72acf09bc531de1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:52:45.650208Z",
     "start_time": "2025-01-27T13:52:45.642088Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:18.483229Z",
     "iopub.status.busy": "2025-01-27T15:33:18.483057Z",
     "iopub.status.idle": "2025-01-27T15:33:18.491490Z",
     "shell.execute_reply": "2025-01-27T15:33:18.490885Z",
     "shell.execute_reply.started": "2025-01-27T15:33:18.483210Z"
    }
   },
   "outputs": [],
   "source": [
    "def training(model,\n",
    "             train_loader,\n",
    "             val_loader,\n",
    "             epoch,\n",
    "             loss_fct,\n",
    "             optimizer,\n",
    "             save_ckpt_callback=None,\n",
    "             early_stop_callback=None,\n",
    "             eval_step=500,\n",
    "             ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "\n",
    "    global_step = 1  # 全局步数\n",
    "    model.train()  # 训练模式\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            for batch in train_loader:\n",
    "                encoder_inputs = batch[\"encoder_inputs\"]\n",
    "                encoder_inputs_mask = batch[\"encoder_inputs_mask\"]\n",
    "                decoder_inputs = batch[\"decoder_inputs\"]\n",
    "                decoder_labels = batch[\"decoder_labels\"]\n",
    "                decoder_labels_mask = batch[\"decoder_labels_mask\"]\n",
    "\n",
    "                # 前向传播\n",
    "                logits, scores = model(\n",
    "                    encoder_inputs=encoder_inputs,\n",
    "                    decoder_inputs=decoder_inputs,\n",
    "                    attn_mask=encoder_inputs_mask\n",
    "                )\n",
    "                loss = loss_fct(logits, decoder_labels, padding_mask=decoder_labels_mask)\n",
    "\n",
    "                # 反向传播\n",
    "                optimizer.zero_grad()  # 梯度清零\n",
    "                loss.backward()  # 反向传播\n",
    "                optimizer.step()  # 优化器更新参数\n",
    "\n",
    "                loss = loss.cpu().item()\n",
    "\n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss,\n",
    "                    \"step\": global_step\n",
    "                })\n",
    "\n",
    "                # 评估\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()  # 评估模式\n",
    "                    # 验证集损失和准确率\n",
    "                    val_loss = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss,\n",
    "                        \"step\": global_step\n",
    "                    })\n",
    "                    model.train()  # 训练模式\n",
    "\n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        # model.state_dict() 返回模型参数的字典，包括模型参数和优化器参数\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), -val_loss)\n",
    "                        # 保存最好的模型，覆盖之前的模型，保存step，保存state_dict,通过metric判断是否保存最好的模型\n",
    "\n",
    "                    # 3. 早停 early stopping\n",
    "                    if early_stop_callback is not None:\n",
    "                        # 验证集准确率不再提升，则停止训练\n",
    "                        early_stop_callback(-val_loss)\n",
    "                        # 验证集准确率不再提升，则停止训练\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict  # 早停，返回记录字典 record_dict\n",
    "\n",
    "                # 更新进度条和全局步数\n",
    "                pbar.update(1)  # 更新进度条\n",
    "                global_step += 1  # 全局步数加一\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "\n",
    "    return record_dict  # 训练结束，返回记录字典 record_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e53ee784c3c7e957",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:53:27.384491Z",
     "start_time": "2025-01-27T13:52:45.651204Z"
    },
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:33:18.492605Z",
     "iopub.status.busy": "2025-01-27T15:33:18.492210Z",
     "iopub.status.idle": "2025-01-27T15:48:11.076906Z",
     "shell.execute_reply": "2025-01-27T15:48:11.076305Z",
     "shell.execute_reply.started": "2025-01-27T15:33:18.492584Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 10999/167400 [14:51<3:31:11, 12.34it/s, epoch=6] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Early stop at epoch 6 / global_step 11000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64\n",
    "train_loader = DataLoader(train_ds, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)\n",
    "test_loader = DataLoader(test_ds, batch_size=batch_size, shuffle=False, collate_fn=collate_fn)\n",
    "\n",
    "epoch = 100\n",
    "\n",
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx),encoder_num_layers=2,decoder_num_layers=2)\n",
    "\n",
    "model.to(device)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失\n",
    "loss_fct = cross_entropy_with_paddings\n",
    "\n",
    "# 2. 定义优化器\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.001)\n",
    "\n",
    "# 3.save model checkpoint\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(save_dir=\"checkpoints\", save_step=500, save_best_only=True)\n",
    "\n",
    "# 4. early stopping\n",
    "early_stop_callback = EarlyStopCallback(patience=5, min_delta=0.01)\n",
    "\n",
    "# 训练过程\n",
    "record_dict = training(\n",
    "    model,\n",
    "    train_loader,\n",
    "    test_loader,\n",
    "    epoch,\n",
    "    loss_fct,\n",
    "    optimizer,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=500\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "43030e8b25455e24",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:53:27.386486Z",
     "start_time": "2025-01-27T13:53:27.385486Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:48:11.078514Z",
     "iopub.status.busy": "2025-01-27T15:48:11.077915Z",
     "iopub.status.idle": "2025-01-27T15:48:11.192775Z",
     "shell.execute_reply": "2025-01-27T15:48:11.192069Z",
     "shell.execute_reply.started": "2025-01-27T15:48:11.078490Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhYAAAGdCAYAAABO2DpVAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAUqRJREFUeJzt3Xd8FEXjBvBnr6YXEhIIJBA6hN6bSEcBBXtBRewKAuKLr6CgiAKWF7FiBxui/sCGiESqoffee0tCgOTSc7mb3x+XXHLJJbkLe9lk7/l+Pvlwtzu7NzcJuSezszOSEEKAiIiISAYapStARERE6sFgQURERLJhsCAiIiLZMFgQERGRbBgsiIiISDYMFkRERCQbBgsiIiKSDYMFERERyUZX1S9otVpx8eJFBAYGQpKkqn55IiIiqgQhBNLT0xEVFQWNpux+iSoPFhcvXkR0dHRVvywRERHJ4Ny5c6hfv36Z+6s8WAQGBgKwVSwoKEi285rNZqxcuRKDBw+GXq+X7bzehu0oD7ajPNiO8mA7ysPb29FkMiE6Otr+OV6WKg8WhZc/goKCZA8Wfn5+CAoK8spvuFzYjvJgO8qD7SgPtqM82I42FQ1j4OBNIiIikg2DBREREcmGwYKIiIhkw2BBREREsmGwICIiItkwWBAREZFsGCyIiIhINgwWREREJBsGCyIiIpINgwURERHJhsGCiIiIZMNgQURERLJRTbB495/jWHJKg0RTjtJVISIi8lpVvrqpp/y84zwuZ2hwLdOM6DCla0NEROSdVNNjUbiMq4BQuCZERETeSzXBopBgriAiIlKMaoKFpHQFiIiISD3BgoiIiJSnnmBR0GXBSyFERETKUU2w4KUQIiIi5akmWBTiXSFERETKUU2wKLzdlIiIiJSjnmBR8C/HWBARESlHNcGiEHMFERGRclQTLHglhIiISHnqCRYF/wpeCyEiIlKMaoJFIcYKIiIi5agnWPBaCBERkeJUEyzssYJdFkRERIpRTbAoxFxBRESkHNUEC14JISIiUp56gkXBxRDeFUJERKQc1QQLIiIiUp5qgkXhpRD2VxARESlHPcGi4F9eCSEiIlKOaoIFERERKU81waLoUgi7LIiIiJSimmAB+10hCleDiIjIi6koWBAREZHSVBMsOEEWERGR8tQTLJSuABEREaknWBTiGAsiIiLlqCZY8K4QIiIi5aknWPBiCBERkeJUEywK8VIIERGRclQTLLhWCBERkfLUEyyUrgARERGpJ1gU4qUQIiIi5agnWHCGLCIiIsWpJ1gU4O2mREREylFNsGB/BRERkfJUEyzs2GFBRESkGPUFCyIiIlKMaoIF57EgIiJSnmqCBRERESlPNcHC3mPBiSyIiIgUo5pgQURERMpTTbAoXN2U/RVERETKUU2wICIiIuWpJlgUjbFQth5ERETeTDXBgoiIiJSnmmBROKU3OyyIiIiUo5pgQURERMpzK1hYLBZMmzYNsbGx8PX1RePGjTFz5szqMXcE57EgIiJSnM6dwm+++Sbmz5+Pr7/+GnFxcdi+fTvGjBmD4OBgjB8/3lN1JCIiohrCrWCxceNGjBgxAsOGDQMANGzYED/88AO2bt3qkcq5QwIXCyEiIlKaW8GiZ8+e+Oyzz3D06FE0a9YMe/bsQUJCAubOnVvmMbm5ucjNzbU/N5lMAACz2Qyz2VzJapdWeAkk32KR9bzeprDt2IbXh+0oD7ajPNiO8vD2dnT1fUvCjUEJVqsVU6dOxVtvvQWtVguLxYI33ngDU6ZMKfOYV199FTNmzCi1fdGiRfDz83P1pSv07j4tTmdIeKy5BW1qsduCiIhITllZWbj//vuRlpaGoKCgMsu5FSwWL16MyZMn4+2330ZcXBx2796NiRMnYu7cuRg9erTTY5z1WERHRyMlJaXcirnrzk82Y88FEz64pzVuah0l23m9jdlsRnx8PAYNGgS9Xq90dWostqM82I7yYDvKw9vb0WQyITw8vMJg4dalkMmTJ+PFF1/EvffeCwBo06YNzpw5g9mzZ5cZLIxGI4xGY6nter1e1m+MRmMbY6HV6LzyGy43ub8/3ortKA+2ozzYjvLw1nZ09T27dbtpVlYWNBrHQ7RaLaxWqzunISIiIpVyq8filltuwRtvvIGYmBjExcVh165dmDt3Lh555BFP1c9lklS4uinHVxARESnFrWDxwQcfYNq0aXjmmWeQnJyMqKgoPPnkk5g+fbqn6kdEREQ1iFvBIjAwEPPmzcO8efM8VJ3Ks68Vwg4LIiIixXCtECIiIpKNaoKFxIk3iYiIFKeaYEFERETKU12w4OqmREREylFdsCAiIiLlqCZYFM5jQURERMpRTbAgIiIi5akmWHAeCyIiIuWpJlgQERGR8lQTLDiPBRERkfJUEyyIiIhIeaoJFkVjLNhnQUREpBTVBAsiIiJSnmqCReE8FuyvICIiUo5qggUREREpTzXBgvNYEBERKU81wYKIiIiUp55gwXksiIiIFKeeYEFERESKU02wkOxdFuyzICIiUopqggUREREpTzXBgmuFEBERKU81wYKIiIiUp5pgwXksiIiIlKeaYEFERETKU02wKBpjwS4LIiIipagmWBAREZHyVBMsCuex4BgLIiIi5agmWBAREZHy1BMsOI8FERGR4tQTLIiIiEhxqgkWnMeCiIhIeaoJFkRERKQ81QSLwnksOMqCiIhIOaoJFkRERKQ81QQLzmNBRESkPNUECyIiIlKeaoKFxHksiIiIFKeaYEFERETKU02w4DwWREREylNNsCAiIiLlqSZYSAWDLARHWRARESlGNcGCiIiIlKe6YMExFkRERMpRXbAgIiIi5agmWHAeCyIiIuWpJlgQERGR8lQTLIoWN2WfBRERkVJUEyyIiIhIeaoJFkXzWBAREZFSVBMsiIiISHmqCRZcK4SIiEh5qgkWREREpDzVBAvOY0FERKQ81QQLIiIiUp5qgoVUMMpCcJAFERGRYlQTLAoxVhARESlHPcFCqrgIEREReZZ6gkUBXgkhIiJSjmqCBTssiIiIlKeaYEFERETKU02wkNhlQUREpDjVBItCvN2UiIhIOW4HiwsXLuCBBx5AWFgYfH190aZNG2zfvt0TdXOLxFEWREREitO5U/jatWvo1asX+vXrh7/++gu1a9fGsWPHEBoa6qn6uY39FURERMpxK1i8+eabiI6OxoIFC+zbYmNjZa9UZXCMBRERkfLcCha///47hgwZgrvuugvr1q1DvXr18Mwzz+Dxxx8v85jc3Fzk5uban5tMJgCA2WyG2WyuZLVLs1qtAID8fIus5/U2hW3HNrw+bEd5sB3lwXaUh7e3o6vvWxJujHb08fEBAEyaNAl33XUXtm3bhgkTJuCTTz7B6NGjnR7z6quvYsaMGaW2L1q0CH5+fq6+dIW+O6bBthQNRjSwoH8UL4gQERHJKSsrC/fffz/S0tIQFBRUZjm3goXBYEDnzp2xceNG+7bx48dj27Zt2LRpk9NjnPVYREdHIyUlpdyKuev5n/fg971JeH5gYzx1Y2PZzuttzGYz4uPjMWjQIOj1eqWrU2OxHeXBdpQH21Ee3t6OJpMJ4eHhFQYLty6F1K1bF61atXLY1rJlSyxZsqTMY4xGI4xGY6nter1e1m+MVmO7wUWr1XjlN1xucn9/vBXbUR5sR3mwHeXhre3o6nt263bTXr164ciRIw7bjh49igYNGrhzGo/iNBZERETKcStYPPfcc9i8eTNmzZqF48ePY9GiRfjss88wduxYT9XPdbwthIiISHFuBYsuXbrgl19+wQ8//IDWrVtj5syZmDdvHkaNGuWp+rmNPRZERETKcWuMBQAMHz4cw4cP90Rdrgv7K4iIiJSnurVCiIiISDmqCRYcYkFERKQ81QSLQlYOsiAiIlKMaoKFn14LAMg2WxSuCRERkfdSTbDQaW1vxWJljwUREZFSVBMstBrbIAsGCyIiIuWoJljoC4JFPoMFERGRYlQTLNhjQUREpDzVBQuzhcGCiIhIKaoJFjr2WBARESlONcFCqy0MFlaFa0JEROS9VBMsdBrbW+HgTSIiIuWoJlhw8CYREZHyVBMszBbbJZDVRy4rXBMiIiLvpZpg8dWGMwCAHLMV565mKVwbIiIi76SaYJGcnmt//PXG08pVhIiIyIupJli8PLS5/fEXCacUrAkREZH3Uk2wqBvso3QViIiIvJ5qgsWVzDylq0BEROT1VBMs9FrVvBUiIqIaSzWfxgwWREREylPNp7GhYEpvIiIiUo5qgkXhlN5ERESkHNV8GmvYYUFERKQ41QQLIiIiUp5qgkWTyACH58mmHIVqQkRE5L1UEywa1PJzeN511irkmC0K1YaIiMg7qSZYOLPt9FWlq0BERORVVBUsHmnGHgoiIiIlqSpY+OmUrgEREZF3U1WwqOcvHJ5bRRkFiYiIyCNUFSx8tY7Pf9l5XpmKEBEReSlVBQupxCRZ/xxKVqYiREREXkpVwaKkjNx8patARETkVVQdLIiIiKhqMVgQERGRbBgsiIiISDaqDxYW3nNKRERUZVQfLLheCBERUdVRXbCICvZxeH7okkmhmhAREXkf1QWLIB/Heb3v/GSTQjUhIiLyPqoLFlqtVHEhIiIi8gjVBYu+zWorXQUiIiKvpbpg8Wy/xkpXgYiIyGupLlhoNaUvhVisAkLwtlMiIiJPU12wcKbx1OW459PNSleDiIhI9bwiWADA1tNXla4CERGR6qkyWOh5ZwgREZEiVBksiIiISBmqDBZlrQ/CWTiJiIg8S5XBoltsmNPt93/OAZxERESepMpg8d597Z1uv5ZlrtqKEBEReRlVBouIQJ+KCxEREZHsVBksiIiISBkMFkRERCQbBgsiIiKSDYMFERERyUa1wSL+uT5KV4GIiMjrqDZYNI0MdLr9WmZeFdeEiIjIe6g2WJSlw8x4patARESkWl4XLIiIiMhzvDJY/LrrgtJVICIiUiVVB4tGtf2dbp/44+6qrQgREZGXuK5gMWfOHEiShIkTJ8pUHXk90iu2zH1JppwqrAkREZF3qHSw2LZtGz799FO0bdtWzvrISiNJZe7rNmtVFdaEiIjIO1QqWGRkZGDUqFH4/PPPERoaKnedZONv1CpdBSIiIq+iq8xBY8eOxbBhwzBw4EC8/vrr5ZbNzc1Fbm6u/bnJZAIAmM1mmM3yLWNeeK7i59RAlHvMtF/3YfqwFrLVQQ2ctSO5j+0oD7ajPNiO8vD2dnT1fbsdLBYvXoydO3di27ZtLpWfPXs2ZsyYUWr7ypUr4efn5+7LVyg+vmieipQcoLy3+O3ms2htPQkfdmyUUrwdqfLYjvJgO8qD7SgPb23HrKwsl8pJQojy/6wv5ty5c+jcuTPi4+PtYyv69u2L9u3bY968eU6PcdZjER0djZSUFAQFBbn60hUym82Ij4/HoEGDoNfr7dvbvvYPss3WMo/b+VI/BProy9zvbcpqR3IP21EebEd5sB3l4e3taDKZEB4ejrS0tHI/v93qsdixYweSk5PRsWNH+zaLxYL169fjww8/RG5uLrRaxz//jUYjjEZjqXPp9XqPfGNKnrd1vWBsO32tzPI6D9WjpvPU98fbsB3lwXaUB9tRHt7ajq6+Z7cGbw4YMAD79u3D7t277V+dO3fGqFGjsHv37lKhojqYNKh5ufvfWHYI3285wzVEiIiIZOBWj0VgYCBat27tsM3f3x9hYWGltlcXPRqHITbcH6dSMp3u/3H7Ofy4/Rzm/XMM214aWMW1IyIiUhdVz7xZ6IP7OlRY5nJ6boVliIiIqHyVut20uLVr18pQDc9qXS9Y6SoQERF5Ba/osSAiIqKqwWBBREREsvGaYNE0IqDCMpm5+VVQEyIiIvXymmDxdN/GFZY5dMlUBTUhIiJSL68JFrd3rI8hcZHllrnzk014dOE2nLni/NZUIiIiKp/XBAsAmD+qU4VlVh1Oxo1vr8XMZQdhtbo82zkRERHBy4KFRiOhfXSIS2W/TDiFJi8tx5ojyZ6tFBERkYp4VbAAXJssq5BVAGMWuLaKKxEREXlhsIiu5YcJA5oqXQ0iIiJV8rpgAQD1Q33dKr/pxBUP1YSIiEhdvDJYjOxQz63y932+GVe5+ikREVGFvDJY6LXuv+35a497oCZERETq4pXBAgB6Nwl3q/yVDPZYEBERVcRrg4VGI7lVfumuCx6qCRERkXp4bbBo5sLaISXlW6weqAkREZF6eG2weG5QMzzaO9atY6b+ss9DtSEiIlIHrw0W/kYdpg1v5dYxP20/j+w8i4dqREREVPN5bbAo1L1RLbfKt5y+AhuOp3ioNkRERDWb1wcLncb9Jhj1xRZYuEAZERFRKTqlK6A0gcoFhMZTlwMAbmxWG+EBRrxzV1tIknt3mhAREamN1weLXk3CseF45afsXnf0MgDAlGNG/MEkrP1PXzQM95erekRERDWK118Kua9LjCzniT+YBADo+85aWc5HRERUE3l9sAj1N+C9e9vj8Rvcu/WUiIiISvP6YAEAI9rXw0vD3Lv1lIiIiEpjsPCA3/dcVLoKREREimCwKEcH6RhQibtGvkw4BQDIMXMyLSIi8i4MFsW8dUdb++N7tauxxPAqXtZ9B3fDhdUqsGzvRbSYtgILNpwqtT8334JvN53G6ZTM660yERFRtcJgUUyLuoH2x/nQQiMJPKb7C8/plrh1nn0X0jBu0S4AwIw/DkIIx2Dy6bqTmPbbAd5BQkREqsNgUUxcVDC6NrRN8f1/lhsx3TwaADBBtxSPa5dV+rynr2Q5PN9yqvLzZhAREVVnDBbFaDUSfnqqh/35N5YheMt8DwDgJf0ijNL+U6nzWqy25dYvpGZj9l+HrmtCLiIiourM62ferMjHlhHwl7IxVvc7ZuoWIFP44Fdr70qdq9ec1TLXjoiIqHphj4UTnz/U2eH52/n3YEH+EGgkgXf0n2CIZptb59t++pqc1SMiIqq2GCycGNQqEquev7HYFgmv5T+In/P7QCdZ8YH+ffTR7HH5fC8u3cfVUImIyCswWJSh5DqlAhq8mP84/rR0hUGy4FP9u+giHXb5fDe/t97p9l1nr2HA/9ZizZHk66gtERFR9cBg4QYLtJhoHofVlvbwlfLwleFttJFOunTs0aQMp9tHf7UVJy5nYswC1y6vXM3Mw7K9F5Gbz8m3iIio+mGwKENZFy7M0OFp80RssrRCoJSNbwxz0Ew6V+nXMeXkl9p29koWftt9AVYnl0/u/WwTxi3ahbnxRyv9mkRERJ7CYFEJuTDgMfPz2GVtglApA98ZZqOhdEm28/d5ew0mLN6NX3ZdKLWvsOdj2R75Xo+IiEguDBaVlAlfjM57AYesMYiQUvGdYTaikHJd50zNysO/xy7bn289dfV6q0lERFSlGCzK4KvXVljGhAA8mDcFJ6x1UV9KwXeGWaiN1Eq/ZvvX4vHgl1srfTwREZHSGCzKEBXii+cHNauwXAqC8UDeVJwX4WikScQ3htkIhvOBmu76cfs5TFm6F8eS0vHb7guYsnSfLOclIiLyFAaLcjw7oClC/PQVlruEMNyf9xKSRAhaas7ha8McBCCrwuNc8cPWcxj07npMWLwbP2w9a99+ITUbu85y4i0iIqpeGCwq8OXoLqgT5IMgn/JnPz8rIvFA3lRcFQForzmJLw3vwAe5Hq3bbR9vRFqW2aOvQURE5A4Giwp0ahCKzVMHYGibuhWWPSbq46G8F2ESvuimOYxP9POgR+nbSeV0OcOz4YWIiMgdDBYuqhPs41K5/aIRHsmbjGxhQF/tHryn/xBaeHIyK4F8ixWX0xkwiIhIeQwWLnqyT2Pc2ak+vhzducKy20ULPG5+HrlCh6HarXhL/xkkWD1Sr4Fz16PJS3+hyxv/4OBFE6xWgRf+bw++TDjlkdcjIiIqD5dNd5GvQYt37mrncvkEaxuMM4/HfP083KH9F5nCB9PzH0bpVUjk88uu87ixWQR+2n4eAFA32AedGoQiMsi13hYiIqLrxR4LD4q3dsYk89OwCgkP6eLxX91ilD1ZuDwy84rGdDzz/U4M+N86j74eERFRcQwWlTBzRJzLZX+39sJL+Y8AAJ7W/YHx2l/gqXDx+b+nsHDDaYdtGbmeHTxKRERUHINFJTzYoyF2ThuEm+LquFT+B8sAzDSPAgBM0v8fvtfPQqyMa4sUt+nkFY+cl4iIyBUMFpVUy9+AUP+KJ88q9KVlGF4xj0a2MKCX9gBWGF7Es9qlMIDzUBARkXowWFyH/wxujhuahrtc/mvLEAzOexPrLW1glMx4Xv9/+NMwFV2kwx6sJRERUdVhsLgOYQFGfPtoN6yf3A9/T+zj0jHnRCQeMr+IZ/PG4bIIQlPNBfxsfA1zdJ/JtsYIERGRUhgsZBAT5ofmdQLdOELCH9aeGJD7Dhbl9wMA3Ktbi1XG/2CEJgGevnOEiIjIUxgsPKh/iwjMvr1NmftNCMDU/MdxZ+50HLXWQ7hkwnuGj/GNfg5ipKQqrCkREZE8GCxkdG+XaIfnQT463NslGkuf6VnucdtFCwzLm423zXcjV+jRR7sPKw0v4Bntbx5fa4SIiEhODBYymjEiDt892s3+XCNJkCQJHWNCKzzWDB0+sozE4Lw3kWCJg49kxgv6H7HMMBWdpCOy1/VUSib+2HMRQvCyCxERyYfBQkZGnRa9m4bD36AFAPRtEVGqzMCWkeWe44yogwfMUzEx7xlcEYForjmPJcYZeEP3JYIqObgzyZSD0ymZDtv6vbMWz/6wC7FTlmPb6atunS/HbMEDX2zBF/+eLLdcZm4+TDm8nZaIyJswWHjA2sn98O2jXXFL29JLrT/cs6ELZ5Dwq7U3BuS+gx/z+wIARulWYZVxMoZrNsHdwZ3dZq1C33fW4lpmntP9d32yCRarwOrDSbiW5bxMcYu3nkXC8RS8/uehMssIIRD3yt9o++pK5Jg9uborERFVJwwWHlA70IgbmtaGJBUtOPbfm1pgZPso9GwcBh+9a82eikD8N/8J3JM7DSesdVFbSsOHhg+wUP8W6kvJbtdrx5lr+DLhFP4+kFhq38KNp/HIwu2445MtFZ4ny4WgkG8tCj/3fb4Ze8+nulVXue06ew3L9l5UtA5ERN6AwaKKPN23Mebd2wEajYQFD3d169gtoiVuzpuDueY7kSt06Kvdg3jDC3hS+wd0bgzufOyb7Zi57CCe/HZHqX3L99mmGD93LRsfHtDAbCl7mfektBz742NJ6RW+7q6zqbj1ww0u1xOwXW4Z8WECXl920K3jynLbxxsxbtEu7L+QJsv5iIjIOQYLBUiVWDk9D3q8b7kdN+fNwSZLK/hKeZii/wF/GF7GzZotMKLiSxjlKf6Be8ykwarDl8ss+/WmM/bHg95dD1OOGceTM9D37TX4efs5AICzMaEXU7Ndrs+K/YnYcz4NXySccvmYklYeSMSkn3YjO6+oh+XMlaxKn4+IiCrGYKGA4rninbvauXXsSRGF+8wv4fm8p3BVBKCl5izmG97DNuMzmK37HN2kQ5BQdm9DWXLzHY9ZsvMCrFbXxnJcTs/F1KX7cPpKFib/314AgHAyDqTnnNU4nuzaANS8cnpMXPXEtzuwdOcFfLa+aJBpZUIdERG5jsFCYXd2qo8wf4ObR0lYYu2DAbnv4OP8W3FR1EKQlIX7dGvwo3Em/jVOxGTdYjSRzle6XmuPpuD91cfsz9ccTsbRMi57CAHk5FtKbXNmzeHkgv0Cz/6wC5N+2m3ft3zfJcxafsjlQOOq5PSiSzfMFUREnuVWsJg9eza6dOmCwMBAREREYOTIkThyRP45FtROKvFncwcX5rlw5hqC8Fb+veiV+z7uzXsZi/P7wiR8UV9KwVjd7/jH+AKWGabiUe2fqI1rbp9/3j+2YLH6cBLGLNyGwe+uR2au8zEdJy8X3c76bvzRMs+p0UhYcyQZsVOW4489F7F05wVkFJzzme934rP1J/HEtzvwQkHPhxyK3+nCHgsiIs9yK1isW7cOY8eOxebNmxEfHw+z2YzBgwcjMzOz4oPJrvSkVNf3F7qABputrfBi/hPokjsfz+SNR7ylE8xCi9aa05im/x6bjePwjX42btP8Cz/kVHzSAl/8exKPLNxufx73yt+lysxfe8IeDgDgvVXH8J+f9zg9n1YCxizYVu5r/nPI+XTmry87iK5v/GO/fTXHbMGhS6YKJ/lavq/4XTBMFkREnqRzp/CKFSscni9cuBARERHYsWMH+vRxbXVPKt+Um1ugdb1gjPqi4ts+ncmFAcut3bHc2h2hMGGYdgtu0yagk+YY+mj3oY92H7LEV/jb2hm/WnojwdoaFmjLPF95c1UUWrKz9CWXZXsvOS27/6LJ6XZXLn8UDuR8b9Ux/PemFnjoq63Yeuoq5t7dDrd3rF/h8QB7LIiIPM2tYFFSWprtToJatWqVWSY3Nxe5ubn25yaT7YPFbDbDbJZvVsbCc8l5Tk9pUMvH/thsNjv8xS1BoGuDYBh1GvuAylkj4zD11wNuv841BOE7yyB8ZxmEGCkJIzUbMFKbgEaaRNym3YDbtBtwWQThD0tP/GLpjX0iFp7+i/7/dpQOIWazGa/+VXaAKfk9PXghDWazGVtP2WYM/X7zGTQO90WDWn7wN5b/I221WKrkZ6Qm/TxWZ2xHebAd5eHt7ejq+5ZEJReLsFqtuPXWW5GamoqEhIQyy7366quYMWNGqe2LFi2Cn59fZV5aFVJyAIMGCDIAnx/WYP8121Wpt7rmw6gF8q3A4hMatAoV6BguMGHTdWXAYgTaSScwUrsBt2o3IkwqGpB53BqFXyy9sczaHWdEJKrLZYMoP4F7Glnw7n5bG7QMseKpllanbfJECwviQm0/0s72P9bcgja1uD4KEZG7srKycP/99yMtLQ1BQUFllqt0sHj66afx119/ISEhAfXrl90N7azHIjo6GikpKeVWzF1msxnx8fEYNGgQ9Hq9bOetCk99v8s+b8SxmYOdljl3LQv955Yd4CpDh3zcoNmH27QJGKzZDh+pKI2eF+HYaInDRmscNllbIQll90opYf/0AWj92iqn+47NHAwhBJpNjy+175NR7THAyRoucqvJP4/VCdtRHmxHeXh7O5pMJoSHh1cYLCr1Z/C4ceOwbNkyrF+/vtxQAQBGoxFGo7HUdr1e75FvjKfO60mSVDSGtqy6N4oIxlcPd8b209ew9dRVbD/j/l0eJeVDhzXWDlhj7YAAZGGIZjtGahPQXXMI9aUU3K1bh7uxDgBwwloXG622oLHZ2hLXIF8orIxOs9aUuU+v1+PNFYed7jt4KQNDWkeVujPHU2riz2N1xHaUB9tRHt7ajq6+Z7eChRACzz77LH755ResXbsWsbGxlaocORrbrzH+OZSEOyoYgNi/RST6t4jE3Z9skr0OGfDDEmsfLLH2gS9y0EVzBD01B9FdcwBtpFNorLmExppLeBD/AAAOWWMKgkYrbLW2RDqq9rJWyQm9invgiy1IOJ7idN/7q4+jXXQIBlSwyiwREVWOW8Fi7NixWLRoEX777TcEBgYiMdF2G19wcDB8fX09UkFv0CEmFPteHYyACgYeVpVs+GC9tR3WW22zggYhE101h9FTcwA9NAfQUnMOLTVn0VJzFo/iL1iEhH2ikb1HY7u1GXJQupeqqpQVKgr9tT+RwYKIyEPc+iSbP38+AKBv374O2xcsWICHH35Yrjp5pUAf17vVejQOw9bTVz1YG0cm+OMfayf8Y+0EAAhDGrprDtmDRiNNItpLJ9BecwLP4HfkCS12iabYZG2FjZY47BZNkAfv6zYkIvJGbl8KIeU9068xagca0atJOKb9uh9x9YLw6bqTFR8okysIxp/W7vjT2h0AUBdX0ENzAD00B9FTewD1pCvoJh1GN81hTNQtRbYwYL9oiP3WWOyzxmKvaISTIgpWhWaUV+OPcXaeBb6GsucjISKqKtWj753cYtRp8UD3BgCA7x7rBgA4kZyBfw4lK1KfSwjDUmsfLLX2AfIFYqRk9NQcsPdo1JZM6CIdRRdN0VTfmcKIg6IB9ltjsdfaCPtErKJhoyb7fc9FjP9hF14e1hKP3dBI6eoQkZdjsFCJD+/viBbTVlRc0OMknBWROGuJxGJLfwACjaWLaCOdQlvNSbTWnEKcdBr+Um6ZYWOftRH2WWM9Fjacrbxak01YvAuAbZbUx25oBItVQKsp+66XPedSkWTKweC4OlVVRSLyIgwWKuGjL+oG790kHHd0qofnfnS+XkfVknBC1MMJUQ+/WnsDADSwolFB2GijOVVh2DhQ7DKKHGGjcK0RNTp/LQuD312Pe7vEYPotrZyWGfHRBgBA/HN90DQysCqrR0RegMFChZ66sTF89NX3koIVGhwX9XFc1Mcv1hsAOA8brQvCRlfpCLpqilbRzRRGHBP1cV6E46IIx0URhosiDBdEOC6JMFxFIMqbNdRxUTL5nLicgehQPxh0yrX9/LUnkJVnwVcbTpUZLAqduZLFYEGkNkkHgPPbgU6jFasCg4WKrJvcF8eSMtC7aTi2V+FdI3JwJWy00ZxEnHQG/lKu7S4UnHB6rhyhxwV74Cj4F2H2bZdEmOz1X7E/EU99twNdY2vhpyd7yH5+V7lzkYcLshGpRE4asO//gF3fARd3ApIWaDYECFTmcieDhYo0CPNHgzB/AOr40CgvbDSWLiFKSkGUdAVRUgrqSVdQV7qCSCkVPpIZjaVLaAznK6wCAN76DxBcv+ArGgiuB/iF4WKOEVsSrejeqjHq1qkD+IQAet8KG/T7LWcAwL4wWkm/77kIc74Vd3RynARt88krWLDhFP57Uws0qh3geuMUU95dLqdSMhFTy8/pmAs1/IwQeS2rFTiTYAsTB38D8nNs2zU6oPnNQF6mYlVjsKAapXjYcMYAMyKlq6gnXUEUHINHlHQF9aQU+Em5QFaK7evSbofjowDcBgDFNudLeuj8QgHfEFvQ8A2B1RiM7UkC4bUj0Si6HnpnXIWPxoo04Q+R2AApuQZorHkAbGM6xv9gG2DZv0UEQv0NAIDfdl/AhMW2FzqWlIHV/+kLALiSkYs1Ry5jWJu65d5CasoxY9TnW8rc327GSqRlm9G/RQS+erhLqf37zptwPDkDj/ZuVO5gTyKqRtIuALsXAbu/A66dLtpeuwXQ4UGg7T1AQG3FqgcwWJDK5EGPcyIS50RZM2sKBCMTvWvn4P2h4dCaLgBp5wDTBVxMvITk5EQEIxPBUiZCNdmQhAU6YQYyk21fBTQAugLAZQAHgScBPGko2PnJTNQFcAsAceQ/0ATWxTd6HRJFLWjXbgEiYoDAulj480lEIBQpCMbJlEy0fuVvbJrSH/d9vhlHkzKw+9w1vD6yTZnv9cet57DvQprDtk0nrtgfp2XbFpVbfdhW77NXsvDNptP2/e/+Yxsku++CCR/c16GCliUixeTnAkf+AnZ9C5xYDYiCJQ0MgUCbO2yBol6natMNyWBBXkZCGgLw5+UATAq/EY1bBkAIgTyLFWt2nMdLv+wvVlYgANkIRiae6x2BJRv3IwiZ6BQhIUKfjfOXLtlDSOG/QcWe6yQrpJxUaHNS0aew42HbOvvZf9ED0AMWIeEyQpAkQnH6wyg8mOqDJG0oMvbURlazvjiZG4S45s0h+YY6/OJw9jvkVErZ3Z/3frYJF9NySm3/Y89FBgui6ijpALDzW2Dvj0B2scusDXoDHR4AWt0KGPyVq18ZGCxUqkUd2+qjfgYtsvLKv73SoNVAr5Ww6PHuiD9wCR+urbpZPJUkBLDu6GWM/morAGDSoGYlSkjIgB8y4If/JAgAcQCAv126qUQgCJmIlFJRR7qKSOkaInENkdI11NFcw8B6ViRfPI1wpEInWVEH11BHugZknEQbnf0UwE8fo3XBU6vWBxr/cMA/DPALR/9sX0BnwVURiKsIwlURiCsiCFcRhCsiEOnwgyh2W66zUEFE1Ux2KrB/ia134uKuou2BdYH29wPtRwFhjRWrnisYLFTK36jD/hlDoNdKyMu3QiNJiHvl71LlPn2wE4bE1YEQApIkoVUdfzTNPYq02m0w/fdD5b5G++gQ7D6X6qF34Hm3fbQB6bn59ue/7b4g49klmBAAkwjAMWfjQU7Z/tHAijCk2QKHZAsekdJVRMIWSCIKttWSMqCx5ACm87YvAI0ANCrnf3C+0OAabGEj78uP8aE+H1dEoH3bVRGEDPja7iQ54QtAAiSNrStE0jg+d9hX8nnxshrb4DGtDtDoAa2h2GO97V9N9b0VmkgRZQ7E1NsGYnZ4EGjc3/Z/qQaoGbWkSilcLdWos/XDd21YC1tPX0XTiAAcS84AALSqa+vZkEr0q9/XJRq3dYxGm1dXOj33sLZ1Ma5fE9z83r+eqr7HFQ8VAHDictWPorZCg8sIxWURiv3l3N1hRB5qS6m4u4UP+kZrcCX5IvzzU7Hz8HGEwYRQKR1hUjpqwYRaUjoCpWzoJCtqIw21pTTg3DkML28pkW9lf2tlk7RFIUOrs4WPwseFAaRgv1ajQ69rqdB+92lBgEGxYFP4M1uJxxqt7UvS2oKQ/XHhl87xefFyDmWLlbMHq2KPpYLHmpLPi5cpeVyJMsVfr/hrOXv9ssoqxZwD5JqA3HTbLZG5JiDHVOzf9ILHBfss+QXff0PRz4HWUPRcU3xfGWXsP0+FjwuDbuGXvqjttPpi31d90TZ7oHaB1QpY8gBLLpCfV+KxC9vSE22XOhwGYrYEOhYMxPQP98i3xpMYLLzI4ie6Y++FNMSG+6PdDOeBobhAHz2ign2cdqF/dH/HStdj5XN9oNdq0O+dtZU+h7fJhQHnRQTmHgLmHgKAZqgX4osL+V2dljciD6EoCBuSqeCxLXSEFYSPWpIJ/siBBKBV3UDbgDAhCgaGiRKPC/bZt5dRzmop+DIDFrPt35KEBci3AKj40owGQDgAZFSm1aiQTtJgOLTQHDA6+WA2uPAB7uTDXKOz3dJoDwppxYJCwTZLntJvvfLsIaQgeGj10ElaDMnJge7whKJw4OxnvDLsAzEfAup1rDYDMSuDwcKLaDQS2keHILuCMRfF/f5sb2w5eRVjF+20bxvYMuK66tGMsz3K4kJqdpn7cmFAIsKQKMJcmjXr9NPDZKxZMUIA1vyikGHJL/plbDEX7bPklShn+8o352DXju3o0KE9dMUvodgn7xCVeywsRSFIWGyvbX9c+JVfdjmHsgXlhNX216uwFnte8G/hl/158f3CSXlLwblKvFZZ2yv4JkvCCi2sQJ5MH4LuMgQCPkGAMajg38Bijwv/Dbb1LJT8GbEU/nWfX+xx4c9JXvllCn+uCsOuNb/g5yy/6LEo4/dh4f5iJAA+AJDv7IACGj2gKwhwOmNBEDMWbdMaAJ2haJveF2g8AGg1AjD4ydTgymKwoHKFBxgxrG1djF1UtO3JG50PHBrRPgoZOfn4z5DmWH04GW//fcRpuZLC/A24klmD/7KhsklS0V+4lSDMZlw8qUX7VkMBfeXO4RWEcBJAisKKOS8Ha1atRL8bb4BeEqU/fC0lP8SdPS75YW623ZFgDwrBJYJC4fZA21/81ZU9qBUG27KDiDkvBwkJCejddwD0Rr9i4aHYvzW4p0EuDBbkkvu6RuOHrecQU8sPXRrWsm9f+VwfDH53PQBg9u1t4Gew/Ui1rBuEDjEh+G7zGUwd2hK931wDAA7TXT/RpxE+W38SXz/SFcM/SKjCd0NqY7UKaJxM8pWVl4+/DySiX/MIhPgZnBypEpJUMLCvjF/pZjOyDeFAaCwDWkkaDQCNLfzqfcsvazbD5HcOCG9WbdvRbLFCp5FKjZurSgwW5JJXbonD4Lg66B7ruM5Gs8hALH6iO4SAPVQU6tk4HD0b2wYenZ5Tuqt96tCW+M/g5g6Ldk0d2gKzlh8GYOvJqBPsgwMXTXK/HZJZvsWKzFwLgv2q/pftzGUH8fuei/hrwg0IDzA67Jv26wEs2XkeHWNCsPSZXlVeN6KqlJZtRq85q9GjcRg+f6izYvXgfV9eqDJB1kevRb/mEU6nmO7eKAw9GlduYa+SK4FKxVYljQ33x9ePdMXMka0dykQEOn54lGfePe1xY7PaqBdSwV8idF1GfLQB7V5bifPXsmQ/t1WUvdT9jjNX8WXCKVxOz8XXG08DAM5cycTirWdxITUbS3babs3deTa1zPNn51nw3eYzuJRW9pgVoprgz72XkJGbj/iDSYrWg8HCC+m1Rd/2sIDq3T0cHmDEg90bOGz7/KHODkFh69QBpY4b2DISp+cMw8gO9bBwTBesf6Gfx+vqzQp7lVbsLz172NGkdDy6cBv2l5h+vCy/7b6AFtP+wvqjlwEAb+7Ros1rq5Ce4zjw8HRKJu6Yv8n+3JRtRr7FihvfXosXl+5DrzmrS537me93oOGLf+L5n/bAVHC+V38/gJd/3Y/B767HrrPXcDiRPWRE14PBwgtpNRI2vtgf6yf3K3X5orp6tHcsAGDykOZoFx2CDS/2x5HXb8KuaYMQEeSDSYOaoUWdortNivfKSJLERbYqsPZIMu76ZCNOXi7/vs4/9lzEWysO40pGrtP956469lgIIXDTvPVYdTgZt328AQCQbMpB/MEkWK3O72SYsHg3csxWPFQwI2pitu17N+3X/bBaBYQQSM3Kw0u/7nM47utNZ3D7/I1l1v3FJXuxfJ8t+CzZeR6TftwNAPhx+zkAQHpOPm77eCNumldz52Yhqg5qxqcKyS6qGl8aaFTbHycvZ+LW9lH2bS8Pa4kHuzdAg7Ci27GMOq198q/xA5pi/ICmaPjin1VeXzV4eME2ALYP9T+e7V1muWcLVmn9eO0JnJg1FFqNhHyL1b7/601nMGNE0aWrh77aisL8YLbYHnSdtcr2mj0b4ok+jcr9WWw6rWi+lV93X8SaI5fti6s5s/d82b0ii7edc3j+z6FkWMoIN4Uz0RLVJL/uknP24MpjsKBq55dnemH/hTT0aFQ0bkOSJDQMr36L7ajNvgtpyM232AMbAFzLzMPSXRdwLCndoezMZQdxPDkDCcdTHLY/unAb4qKC8P7q46XO/0+xa78LN57Gwo2ncWr2UORbBfRaDfLyraWOKa68UFEZw9533jux+eTVSo8bqu72X0hDVp4FXWNrVVyYapStp69WXKgK8FIIVTvBvnr0ahLu9PbBihROUX5HRyfrc5BLRny4weH5mIXbMHPZwVJ/8S/ceLpUqACAVYeTnYYKAHjsm+2ltsVOWY6mL/2FtCwzXi5xecPTDiemO91+3+ebq7QeckvJyMWguevw+frSCwoO/yABd3+6CcnpXJSOPIM9FqQqS57uidNXMh3GW5B7in/YmnLMVbbQ3MMLt2JXOXdvUMUyc/Pxxb+n8O4/RwEAbyw/hMf7NHJaNiktFxGBPlVZPfISDBZUrei013dd29egRcuCXguqvEOXTAgPMGL6b/ur7DXVFCrMFiv2nk9Du/rB0GmrrmP4tT8O2gejFko25cCcn48V5yR0NDnvpdh/IQ0XU7Nx4nIm7u8Wg2Df6jn5E9UMDBZULTzbvwlWH07GPV2ila4KATV61drqYPpvB/DD1rN4pFcsHr0hFgEGneyTh63Yfwm7zqViys0t7dviD5Wev6BwsCygxYmvi9b8ybda8fn6k/AxaDHt16IA+eaKw/YJ7dJzzNBrNfDRV+MpuV1w4nIGZv15CM8OaIr20SFKV0f1GCyoWnh+cHM8P7i5R19j2vBWmLnsoEdfg7zDwYsm5FmsTj+kNp24gh+2ngUAfLXhFL7acAqAbbK2Ee2jcDEtB6sPJ+OuTvXL/MA+mpSO/608grH9mqBt/dKvAQBPfWcLCU0jAnFnJ9uYoqsVrLlzNLnoduLbPi771tynvt2BRrX98fHaEwCcz5xbkzy6cBtOX8nCqsPJNf69uErJO5s4eJO8xiO9Gl73OfydzDzqjgkDml53HUhZQggMff9fjPxoA9Kyiu5S2X8hDS//uq/MgZ8Tf9yN+INJuHneekz7dT/eKVikr/usVWj44p9INuVg2d6LuJyei8HvrsffB5Jw64cbkJdvxapDSfYJvU5ezsDIj4oG2C4tmF1UTisOJNpDhafkW6z499jlUhOfecK5a+7Pqrr7XCo+XH0MZkv5dypVV/ll3EpdFRgsyGtUJr1/8kBH++Nx/ZrgwGs3Yf+MIS4dO7NT6bWVS05h7mldG/KWQjlZrALFf19fLjZR2PAPEvDd5rPlHr/vQhpMObafi3+PpSAv34rEgnEPXWetwrhFu3BLiQX5hr7/Lx79ejvGLNiGFfsT0f9/6xwG1G48cQXJphwcT3Z+h4scPDE/zGf/nsSDX27FqC+2uHzM1lNX0eetNVhzJNmt13Llf35uvgVCFH1zR360Ae+sPIq4V/5267WIwYK8zHv3tq+wTKNwf3RpGIrbOtTDTa3rltpf1qRKxUUGGhFkAKYNawFdsdtmJQl484425R771p1tnW6fdZvz4757tBt89M7/Kwso91eL2qw5nIyW01fgt91FkxBtOpGC15cdxPiCicMq8kGx23CPJKWj2ct/lSqTWGKA5fGCyxc7zlzDU9/tcHrerrNWYeDc9S7Voar9fSARP5W4VRkAluyw9bTsPZ+Ghi/+iTf+LH2ZsvgHPWC7Dfjs1SyMKZjQzVVl/U1htQqcuZKJKxm5aPPKSjz2denbofPyrThwsWjiNSGAC6nVf10ZoeB/fY6xIK8yon09tK0fgoFz1+GBbjE4czULa49ctu8f268xxg9o6jBBVKHCX05lfYgfff1mbDl1BUadFm2jAvD3ir/wUPcYjO4ZiyYv2T5ANJKEe7rEoF10CJJMuejSMBRCAL56LRpNXQ7A9su0S8NQbDt9zeH893eLwdRfHOd5iH+uD5pGBmLXtMFoOX1FqToJAfz4RHdsOnkF8/45BgDQaSS8e097+yya5JoxC20fZpN+2mPfNu23A0pVp8oduJiGuKhgt4978ltbGDp+OQMJx1LwzaNdER5gdFizCAA+//cUXhrWyv48L9+KER9tQMs6gZh7T3sAroV6Z2yLG9qOvZCajRBfPfyNOvx3yV78vKPoUtKqw8l46KutyC2x6F1WXtHzX89oMHHzv5g2vJV9qYHqSMk/KhgsyOvEhvvj8MyboNdqIITAuavZ+GbTaYzu2RDRtfzKPK7wjx6jTovGtf1x4nKmw36DToMbmtYGAJjNRdeNi99uWHiOFnWC0KKO89exWIEFY7rifyuP4NZ2UagT7INa/rbF4hqE+eHMFdt6HOEBRjSNtM3X4WvQYte0Qbh9/kacSimqV6Pa/ujWKAzdGoVhfP+mOH0lE7Hh/pAkCR+tOV7mBFEEzPjjAEb3aMgZXwucv5ZdqWBR6LOCybre++cYZo5sXeGt5RuOp+DQJRMOXTLZg0V5svLykZKeh0Vbz+KWdnXtdbVYBfKKjZMoXJzO36BFZl7pVXMLF78rLsdsweaTV9AuKgBrL9n+P89afqhaBYuS6/ewx4KoihX+tSRJEmLC/PDy8FYVHOGoXf0Qe7AIDzDgpWEtyy0/qlsMVh1Kxr1dYio+d3QwAow6vHJLXKl9y8ffYL/mOyQu0mFfqL8BLw1taZ/dckhcJF4aWvS+NBoJjWoH2J//Of4GNC7oJaHSFmw4jW83nUG3RrU4kytQ5qJx7vp28xkMb1sX+y+UXkX26e92QAhg/gMdHf7i/mz9CQxt43hZ0moVOHUlE18lnEJiWg5WHS4ad/HJuhP2uz/KmrbdWagoy4Nf2hbEG9+vsX1bZXtPPKXT6/8oXQU7BgsiFxUfePnCTS1w4KIJD/ZogFHdYiocGPrGbW0wc4Qod5ryDS/2R2Ja+X8V+ht1WD+5H9Ydu4w7nXzY9W1eGwNbRqJd/WA8W8EdKFzxtWL5VoENx69gw/ErSldFcXJ+jt7zmfM7Z/7ab1t99kqJ22ZnLT+Mz9afctjWqIJQvOdcKlYdSpK1V+79NfLdKWMt6Enx1Bwh7LEgqsYmD2mOvw8kYnTPhvZtdYJ98Pdzfdw6T0Vrn9QL8UU9F1adjQnzw4NhDZzu02k1+GJ0Z7fqRSSn0ymZqBfqW2oMhTucBYKUEl39FRnx0YaKCynotvkbceBCGnZOH4Qgn+ubPC3fyS2xHGNBVI2N7dcEY/s1UboaRIqqF2oLvfEHk3DokgmP9I7F4UsmdIwJtYfmd/4+gg/X2O582Tp1AFIruRrtf5dU7WJ0VeXlX/chyEePF25qgT0FtwxvPH4FN7UuY8CVi5buLL1cupJXahgsiKjGeKhHA3yz6YzS1fBKViGw5nAyHi8YwzM33rbQ2c2t6+ClYS3x2NfbHXoZiqYS9265+RYYdVqcvZJln+fkP8VmGf732GUMaBnh0MOTlmVGoI/O5RWeVxxILLXtUmq2fXB3VeM8FkRU5db8p6/bx+yZPhivjWiNHx7v7tZxU25u4fB8QIsIt1+bgM0nr9hvuS3ur/2J6P3mGt5hVExuvgVXM/Mw9Zd9aP7yChxPTkeexflg0e+3nMXHxcZuHE1KR7vXVuKuTzdh1aEk5JidH2e1ClwsmE9j9eHSE4aVd4ebpzFYEFGVKFzKvkNMCGLD/bFl6gCEBxjs+2fd1sbh+e7pg7Du+Rsws1M+Vk/qbV/Eq0fjMJyeMwy7pg1Cq3JWsu3SMBSHZ95kX0cDsK3XMe/e9vhn0o3o2TgM93aJxpHXb8If43rL/XZV560VR5SuQo1x41tr0XFmPBZtsfVQ/G/lUYf9Ja9SLCk2LXvhMTvOXMOjX29H37fX4uEFW3G24DZzAPh5+zl0m70KPeesdjql+z2doxVdOI6XQoi82APdYyqchtpV79zVDnvPp2LlgSR882hXDH7XcSbI6cNboUNMKIwFd9dEBvngrwl9cPN763Fru3q4v1sMTqVk4PN/TyHIR4cQPwP89RKCDEB0aOm/vkL9DXiiTyNM/HE3BrSIcLjdEAC+eaQbfPRa+Oi1+OWZnvAz6NC8INwE+uixqFjPR5v6lZ+fgbzXq78fwIj2UegQE+qwveTsqX/tT3QYR1FysbizV7OQY7Yg3yqw93xqqXMlmnLwxLfbMa5/E4xb5DixXfEJ2wp1b6zsVP4MFkRebMatrXFf1xiEBxgxd+VRLNl5Hnd0rA8/oxZ3d47GLR8kOCxm9OH9HUr9YgNsvQN3dqqPOzvVx6u3xJW6Nvz4DbHo2SS81HG1A43YOnWgvfzzg5sjJswf/V28XDGyQz20rR+MmFp+9tlNo4J98OSNjeFbbMG4kr/4ndnwYn/75ElErli48TQWbjyNk7OGVjgeYsLi3fbHXd4oPefE1KX7sHRX6UGYhQ4npjv9v+fMtUzPL+xWHl4KIfJiWo2EuKhgRAb54M072+L4rKF48862eOWWOLSsG4QlT/d0KD+8bZT98bThrTBpUDPoNBKmDy+azKvwF+yCh7ugS8NQrP1PX4epmksq/gvZR6/Fg90buHTbbaFGtQMcZjd9cWhLh1uDXVUvxBd/juclEXLf8A8ScDQpHXvPp5Za38RV5YUKdyk9dRd7LIioTO2iQ0pte6hHA6w6lIy7OtdHkI8eT/dt7HTOgn4tItCvCgdKfjm6M3acuYbhbUovHOcqw3XMveCuacNbYeay0gtvXa+IQCOS021zPtQL8XVrwax/Jt2ILxNO4Yettstj/77QDze8tUb2OqrNwUsm+6W/uzsrP0urXLOkVhZ7LIioXF881BmhfnosGNMFAPDaiNZI+G8/+6Q+1zMRkpwGtIzECze1cPkWPWeu59iyfHh/Bzx+Q+k1JUquM3F6zjAMKycU/T6uV7mvc3PrOjg9Zxi+f6ybfVtZC+YVKj6bLAA0iQjA7Nvb4MSsoTj02k2K3llQU/20vfRgyqpmUXLaTTBYEFEFBraKxM5pg9CveVHvQ0VTmNdUvjKPpD/2xs0Y3jaq3EtBxRXeOVPozTva2B+3rR/isG/KzS3w4xPd7XV+ok8jAI5LhBcGlZhavugd6Tg745/je2Pfq4PxQHfb+jWPFQs6Wo3kMEaFahal1zHhpRAiqpBag0RJUW6M7TDqNMjNLz2VcqH20SEOvTmv3NIKM/6wXfoouajZwwVjQh7u1RD/iz+K2oFGbJ4yACcuZ5R5/idvtC2IdfC1ITDl5CPYt3Ba6KLv1TP9mqBF3SB0jA7C5rX/4H9j+iPDLJCYlmNfk+bVW+Jwd+fo61q5tKYa1qYuPry/A4a+n4BDl0ovilZT5VvYY0FEVG0cnnlTufuDffVY8nTPCqd5n/9AR4fnY3rF4uSsoVj2bG+HngjAdncMYLsN9vScYdj20kBoNRKaRQZi4sCmmHWbrfyCh7ugUbg/lj5TNKhWkqRioQII8in6e1Gv1WBom7oI8zdAkoBa/gY0rh2AXsXu0NFpNWhbP0SxRekOvjYEb93R1mFbiF/R+wnzN5Q8xKklT/fEO3e1Q7/mte3bYiq4lPPRqI6QJAnLx/fGyVlDnZYpPsHagVcG4uGmrq+KqhSlL4Wwx4KIqBgfvRan5wyDxSoclpUPDzAgJSMP3z/WDa3rBaNxbX/8sPUs4qKC8c+hJADAyuf62AfxhfqV/kDUaCS0rlfUM/DBfR2w8mBSqfEWxU0c2Mz+2JUBsRFBPnh9ZGv4GbTVfgXb/93VDn4GHe7uEo1z17Lgo9dibL8mSM8xo82rKwEAq5/vi3av2R5/8kAnPPXdDodz3N25Pu7sFI1ODULRqUEo7uhYDycuZ2DN4cu4t2u0/TyfPNART323EwBwa7so3NMl2n4OSZIgScAjvWLx1YaiVVTXTe6LiEAf/HssBf1bRMCg06BBoNL3XFRM6cGbDBZERE5oNRKWj78BQ9//FwCwbnI/JJly0Kh2AAAgxM+AjS/2hyRJWLrzPEL9DWgWGYjl42+AgHBp5sNb2kXhlnZRFZZz1wPdna9+Wxk3t65jX85cbsPaFg1Wfb7Y+hmBPnp8+mAnSACC/fQ4OWsosswWBBh1iH+uD2r5G9DpddtcED0ah6FrbNGEUJIkoUlEIJpE2MarrH7+Rpy/lo0+zWrj93G9EOJrQEyY856MRrX97Y/7NKuNBmG2598VDIg1m82oZZTnvXtSPoMFEVH1VHiJAgD8jTp7qChUOPbk9mJjJlpFlT3NeE0054626NQgFK//eajCsre0i8Ifey6Wub9rw1ro3TTcvoCZUVf21fghcUUzVWo0EgKMto+rwoW1vnq4M7acuopb29Urt06NagfYv28lB8CWdHfnaKw9chlWIfDh/R3KLVudXe9qqdeLwYKIqAy1A4349tGu8PPiOySCffV47IZG6BATgjvmbyq1/9n+TfDB6uO4oWk4GjgZ01AvxBeZeflIzTJj+i2t0LpeMJ7p27jg8kPlL9X0bxGJ/i0iK328MwadBl+M7izrOT2td5NwJBxPgVYjYee0QbiWmYeG4f4VH+hBDBZEROW4oWntigt5gU4NauHU7KH4afs5/LHnEhKOp2DBmC7o1zwCT93YGH4GLTJy87HnfCosVoHpt7TC5hNXcFPruvA3apFkykWTCFvPga6azH1SWZ/c3x5PLdrtcnlXJirzM2gxfkBTtKsfgvs+31xq/31dozHrtjZYdSgZep0GH64+hpSMPHz7aFfsu5CGhuH+CPLROwzkVQqDBRERuUSSJNzTJQb3dIlBdp7FPteFf8FlikAfPb59tGiCrhZ1ii4LBfoo/4EnlwEtXZ9R9tVbWuHhXrFITMvB238fcVjJFLDd9bJmcl/7hHMAsHBMFzy8oGiJ+uFt62LK0JaQJAkDW9l6aW5sVhR4K7rEU9UYLIiIyG3ePoHWFw91xmPfbHe6r1/z2vhidBdk5uXbA0OdYB+8c1dbjOwQheaRgcjMs+BoUrrDWJJCfZtHYP6ojnj6+52YOrQFnujT2KPvRW4MFkRERG4a2CoSG17sj8e/3o5Zt7dBWrYZHWNCEGDU2ceOBJXopZEkyeHSWmw5YyFublMXh167qUYGOAYLIiKiSqgX4ovlE27w2PlrYqgAOPMmERERyYjBgoiIiGTDYEFERESyYbAgIiIi2TBYEBERkWwYLIiIiEg2DBZEREQkm0oFi48++ggNGzaEj48PunXrhq1bt8pdLyIiIqqB3A4WP/74IyZNmoRXXnkFO3fuRLt27TBkyBAkJyd7on5ERERUg7gdLObOnYvHH38cY8aMQatWrfDJJ5/Az88PX331lSfqR0RERDWIW1N65+XlYceOHZgyZYp9m0ajwcCBA7Fp0yanx+Tm5iI3N9f+3GQyAQDMZjPMZnNl6uxU4bnkPKc3YjvKg+0oD7ajPNiO8vD2dnT1fbsVLFJSUmCxWBAZGemwPTIyEocPH3Z6zOzZszFjxoxS21euXAk/Pz93Xt4l8fHxsp/TG7Ed5cF2lAfbUR5sR3l4aztmZWW5VM7ji5BNmTIFkyZNsj83mUyIjo7G4MGDERQUJNvrmM1mxMfHY9CgQdDr9RUfQE6xHeXBdpQH21EebEd5eHs7Fl5xqIhbwSI8PBxarRZJSUkO25OSklCnTuk15QHAaDTCaDTanwshAADZ2dmyfmPMZjOysrKQnZ2N/Px82c7rbdiO8mA7yoPtKA+2ozy8vR2zs7MBFH2Ol8WtYGEwGNCpUyesWrUKI0eOBABYrVasWrUK48aNc+kc6enpAIDo6Gh3XpqIiIiqgfT0dAQHB5e53+1LIZMmTcLo0aPRuXNndO3aFfPmzUNmZibGjBnj0vFRUVE4d+4cAgMDIUmSuy9fpsJLLOfOnZP1Eou3YTvKg+0oD7ajPNiO8vD2dhRCID09HVFRUeWWcztY3HPPPbh8+TKmT5+OxMREtG/fHitWrCg1oLMsGo0G9evXd/dlXRYUFOSV33C5sR3lwXaUB9tRHmxHeXhzO5bXU1GoUoM3x40b5/KlDyIiIvIeXCuEiIiIZKOaYGE0GvHKK6843IFC7mM7yoPtKA+2ozzYjvJgO7pGEhXdN0JERETkItX0WBAREZHyGCyIiIhINgwWREREJBsGCyIiIpKNaoLFRx99hIYNG8LHxwfdunXD1q1bla6SYmbPno0uXbogMDAQERERGDlyJI4cOeJQJicnB2PHjkVYWBgCAgJwxx13lFoD5uzZsxg2bBj8/PwQERGByZMnl5off+3atejYsSOMRiOaNGmChQsXevrtKWLOnDmQJAkTJ060b2Mbuu7ChQt44IEHEBYWBl9fX7Rp0wbbt2+37xdCYPr06ahbty58fX0xcOBAHDt2zOEcV69exahRoxAUFISQkBA8+uijyMjIcCizd+9e3HDDDfDx8UF0dDTeeuutKnl/VcFisWDatGmIjY2Fr68vGjdujJkzZzqs28B2LG39+vW45ZZbEBUVBUmS8Ouvvzrsr8o2+/nnn9GiRQv4+PigTZs2WL58uezvt1oQKrB48WJhMBjEV199JQ4cOCAef/xxERISIpKSkpSumiKGDBkiFixYIPbv3y92794thg4dKmJiYkRGRoa9zFNPPSWio6PFqlWrxPbt20X37t1Fz5497fvz8/NF69atxcCBA8WuXbvE8uXLRXh4uJgyZYq9zMmTJ4Wfn5+YNGmSOHjwoPjggw+EVqsVK1asqNL362lbt24VDRs2FG3bthUTJkywb2cbuubq1auiQYMG4uGHHxZbtmwRJ0+eFH///bc4fvy4vcycOXNEcHCw+PXXX8WePXvErbfeKmJjY0V2dra9zE033STatWsnNm/eLP7991/RpEkTcd9999n3p6WlicjISDFq1Cixf/9+8cMPPwhfX1/x6aefVun79ZQ33nhDhIWFiWXLlolTp06Jn3/+WQQEBIj33nvPXobtWNry5cvFSy+9JJYuXSoAiF9++cVhf1W12YYNG4RWqxVvvfWWOHjwoHj55ZeFXq8X+/bt83gbVDVVBIuuXbuKsWPH2p9bLBYRFRUlZs+erWCtqo/k5GQBQKxbt04IIURqaqrQ6/Xi559/tpc5dOiQACA2bdokhLD9Z9RoNCIxMdFeZv78+SIoKEjk5uYKIYR44YUXRFxcnMNr3XPPPWLIkCGefktVJj09XTRt2lTEx8eLG2+80R4s2Iau++9//yt69+5d5n6r1Srq1Kkj3n77bfu21NRUYTQaxQ8//CCEEOLgwYMCgNi2bZu9zF9//SUkSRIXLlwQQgjx8ccfi9DQUHvbFr528+bN5X5Lihg2bJh45JFHHLbdfvvtYtSoUUIItqMrSgaLqmyzu+++WwwbNsyhPt26dRNPPvmkrO+xOqjxl0Ly8vKwY8cODBw40L5No9Fg4MCB2LRpk4I1qz7S0tIAALVq1QIA7NixA2az2aHNWrRogZiYGHubbdq0CW3atHFYA2bIkCEwmUw4cOCAvUzxcxSWUVO7jx07FsOGDSv1PtmGrvv999/RuXNn3HXXXYiIiECHDh3w+eef2/efOnUKiYmJDu0QHByMbt26ObRlSEgIOnfubC8zcOBAaDQabNmyxV6mT58+MBgM9jJDhgzBkSNHcO3aNU+/TY/r2bMnVq1ahaNHjwIA9uzZg4SEBNx8880A2I6VUZVt5g3/1wvV+GCRkpICi8VSahG0yMhIJCYmKlSr6sNqtWLixIno1asXWrduDQBITEyEwWBASEiIQ9nibZaYmOi0TQv3lVfGZDIhOzvbE2+nSi1evBg7d+7E7NmzS+1jG7ru5MmTmD9/Ppo2bYq///4bTz/9NMaPH4+vv/4aQFFblPd/ODExEREREQ77dTodatWq5VZ712Qvvvgi7r33XrRo0QJ6vR4dOnTAxIkTMWrUKABsx8qoyjYrq4za2hSo5CJkVHOMHTsW+/fvR0JCgtJVqVHOnTuHCRMmID4+Hj4+PkpXp0azWq3o3LkzZs2aBQDo0KED9u/fj08++QSjR49WuHY1x08//YTvv/8eixYtQlxcHHbv3o2JEyciKiqK7UjVSo3vsQgPD4dWqy01Gj8pKQl16tRRqFbVw7hx47Bs2TKsWbPGYan6OnXqIC8vD6mpqQ7li7dZnTp1nLZp4b7yygQFBcHX11fut1OlduzYgeTkZHTs2BE6nQ46nQ7r1q3D+++/D51Oh8jISLahi+rWrYtWrVo5bGvZsiXOnj0LoKgtyvs/XKdOHSQnJzvsz8/Px9WrV91q75ps8uTJ9l6LNm3a4MEHH8Rzzz1n71FjO7qvKtusrDJqa1NABcHCYDCgU6dOWLVqlX2b1WrFqlWr0KNHDwVrphwhBMaNG4dffvkFq1evRmxsrMP+Tp06Qa/XO7TZkSNHcPbsWXub9ejRA/v27XP4DxUfH4+goCD7h0SPHj0czlFYRg3tPmDAAOzbtw+7d++2f3Xu3BmjRo2yP2YbuqZXr16lbnc+evQoGjRoAACIjY1FnTp1HNrBZDJhy5YtDm2ZmpqKHTt22MusXr0aVqsV3bp1s5dZv349zGazvUx8fDyaN2+O0NBQj72/qpKVlQWNxvFXtlarhdVqBcB2rIyqbDNv+L9up/ToUTksXrxYGI1GsXDhQnHw4EHxxBNPiJCQEIfR+N7k6aefFsHBwWLt2rXi0qVL9q+srCx7maeeekrExMSI1atXi+3bt4sePXqIHj162PcX3io5ePBgsXv3brFixQpRu3Ztp7dKTp48WRw6dEh89NFHqrtVsrjid4UIwTZ01datW4VOpxNvvPGGOHbsmPj++++Fn5+f+O677+xl5syZI0JCQsRvv/0m9u7dK0aMGOH0lr8OHTqILVu2iISEBNG0aVOHW/5SU1NFZGSkePDBB8X+/fvF4sWLhZ+fX429TbKk0aNHi3r16tlvN126dKkIDw8XL7zwgr0M27G09PR0sWvXLrFr1y4BQMydO1fs2rVLnDlzRghRdW22YcMGodPpxDvvvCMOHTokXnnlFd5uWt198MEHIiYmRhgMBtG1a1exefNmpaukGABOvxYsWGAvk52dLZ555hkRGhoq/Pz8xG233SYuXbrkcJ7Tp0+Lm2++Wfj6+orw8HDx/PPPC7PZ7FBmzZo1on379sJgMIhGjRo5vIbalAwWbEPX/fHHH6J169bCaDSKFi1aiM8++8xhv9VqFdOmTRORkZHCaDSKAQMGiCNHjjiUuXLlirjvvvtEQECACAoKEmPGjBHp6ekOZfbs2SN69+4tjEajqFevnpgzZ47H31tVMZlMYsKECSImJkb4+PiIRo0aiZdeesnhFke2Y2lr1qxx+vtw9OjRQoiqbbOffvpJNGvWTBgMBhEXFyf+/PNPj71vJXHZdCIiIpJNjR9jQURERNUHgwURERHJhsGCiIiIZMNgQURERLJhsCAiIiLZMFgQERGRbBgsiIiISDYMFkRERCQbBgsiIiKSDYMFERERyYbBgoiIiGTDYEFERESy+X/AmO3fs352nwAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 画图\n",
    "plt.plot([i[\"step\"] for i in record_dict[\"train\"]], [i[\"loss\"] for i in record_dict[\"train\"]], label=\"train\")\n",
    "plt.plot([i[\"step\"] for i in record_dict[\"val\"]], [i[\"loss\"] for i in record_dict[\"val\"]], label=\"val\")\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "caf2b00ee2704d39",
   "metadata": {},
   "source": [
    "# 推理\n",
    "\n",
    "- 接下来进行翻译推理，并作出注意力的热度图"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "913a1278f74da64b",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:48:11.193859Z",
     "iopub.status.busy": "2025-01-27T15:48:11.193525Z",
     "iopub.status.idle": "2025-01-27T15:48:11.667506Z",
     "shell.execute_reply": "2025-01-27T15:48:11.666414Z",
     "shell.execute_reply.started": "2025-01-27T15:48:11.193831Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx),encoder_num_layers=2,decoder_num_layers=2)\n",
    "model.load_state_dict(torch.load(\"checkpoints/seq2seq_layer_2.ckpt\",weights_only=True, map_location=device))\n",
    "\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()  # 切换到验证模式\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "\n",
    "    def draw_attention_map(self, scores, src_words_list, trg_words_list):\n",
    "        # 绘制注意力热力图\n",
    "        # scores.shape = [source sequence length, target sequence length]\n",
    "\n",
    "        # 注意力矩阵,显示注意力分数值\n",
    "        plt.matshow(scores.T, cmap='viridis')\n",
    "        # 获取当前的轴\n",
    "        ax = plt.gca()\n",
    "\n",
    "        # 设置热图中每个单元格的分数的文本\n",
    "        for i in range(scores.shape[0]):  # 输入\n",
    "            for j in range(scores.shape[1]):  # 输出\n",
    "                # 格式化数字显示\n",
    "                ax.text(j, i, f\"{scores[i, j]:.2f}\", va='center', ha='center', color='k')\n",
    "\n",
    "        plt.xticks(range(scores.shape[0]), src_words_list)\n",
    "        plt.yticks(range(scores.shape[1]), trg_words_list)\n",
    "        plt.show()\n",
    "\n",
    "    def __call__(self, sentence):\n",
    "        # 预处理句子，标点符号处理等\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_inputs, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()],\n",
    "            padding_first=True,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )  # 对输入进行编码，并返回encoder_inputs和encoder_padding_mask\n",
    "        # 转换成tensor\n",
    "        encoder_inputs = torch.Tensor(encoder_inputs).to(dtype=torch.int64)\n",
    "        # 由输入得到预测结果，\n",
    "        # 注意，这里输入的样本只有一个，符合Seq2Seq模型的生成方法要求batch_size=1\n",
    "        preds, scores = self.model.infer(encoder_inputs=encoder_inputs, attn_mask=attn_mask)\n",
    "\n",
    "        # 通过tokenizer转换成文字\n",
    "        # trg_tokenizer.decode方法要求输入的是字符串列表，所以需要[preds]\n",
    "        # 方法返回字符串列表，所以需要[0]\n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=False)[0]\n",
    "\n",
    "        # 对输入编码id进行解码，转换成文字,为了画图\n",
    "        src_decoder = self.src_tokenizer.decode(encoder_inputs.tolist(), split=True, remove_bos=False, remove_eos=False)[0]\n",
    "\n",
    "        self.draw_attention_map(\n",
    "            # [1, sequence length, sequence length] \n",
    "            # -> [sequence length, sequence length]\n",
    "            scores.squeeze(0).numpy(),\n",
    "            src_decoder,  # 注意力图的源句子\n",
    "            trg_sentence,  # 注意力图的目标句子\n",
    "        )\n",
    "        return \" \".join(trg_sentence[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3aa03683664d5b62",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-01-27T13:53:27.389488Z",
     "start_time": "2025-01-27T13:53:27.388488Z"
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:48:11.668481Z",
     "iopub.status.busy": "2025-01-27T15:48:11.668262Z",
     "iopub.status.idle": "2025-01-27T15:48:11.841778Z",
     "shell.execute_reply": "2025-01-27T15:48:11.841216Z",
     "shell.execute_reply.started": "2025-01-27T15:48:11.668460Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAbkAAAGkCAYAAACsHFttAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAiMtJREFUeJzs3XeYXOV58P/vOdN2+u7O9iKttKveu4QoAolqsMFAXDFOXGL7NbHBsROcXwIuCYljYjv260acgJPXNh1jOggQSKDe++5qe+9Tdvo5vz9WmtVoZ1VgZxZN7s91zYXmzH3OPPecmeee5znPLIqu6zpCCCFEFlInuwFCCCFEukiRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkTvNunXrUBQFRVHYu3fvpLShsbEx0YbFixdP6LHXrVvH17/+9Qk9Zjarqqrixz/+8WQ3I0HXdb74xS+Sn59/1veooig888wzGW3bB839998/4Z+fbPZB6PvS1QYpcmf4whe+QEdHB/Pnz08qOIqiYDabqamp4fvf/z5n/jW0Q4cO8Wd/9mcUFhZisViYOXMm//AP/8Dw8HBS3L59+/jwhz9MUVEROTk5VFVV8bGPfYzu7m4AKisr6ejo4Bvf+EbGchYXh5deeomHH36Y5557LvEeTaWjo4Prr78+w637YPnrv/5rNm7cONnNuKicre87/bZ169bEPsFgkPvuu4+ZM2disVgoKCjg9ttv59ChQ0nHHh4e5t5776W6upqcnBwKCwu54oor+OMf/5iIeeqpp9i+ffuE52Wc8CNe5Gw2GyUlJUnbXnvtNebNm0c4HGbz5s18/vOfp7S0lM997nMAbN26lQ0bNrBhwwaef/55iouL2b59O9/4xjfYuHEjb7zxBmazmZ6eHtavX8+NN97Iyy+/TG5uLo2NjTz77LMEAgEADAYDJSUlOByOjOcuPtjq6+spLS3lkksuSfl4JBLBbDaPef/+b+RwOOQzdIHO1vedzuPxABAOh9mwYQPNzc08+OCDrFq1iq6uLh544AFWrVrFa6+9xurVqwH40pe+xLZt2/jpT3/K3Llz6evr45133qGvry9x3Pz8fLxe78QnpouEK664Qv/a176WuN/Q0KAD+p49e5Li1q9fr3/lK1/RdV3XNU3T586dqy9fvlyPx+NJcXv37tUVRdH/+Z//Wdd1XX/66ad1o9GoR6PRc7blvvvu0xctWvS+8jnTFVdcod911136N7/5TT0vL08vLi7W77vvvsTjDz74oD5//nzdZrPpFRUV+pe//GXd5/MlHWPz5s36FVdcoVutVj03N1e/5ppr9P7+fl3XdT0ej+v/9E//pFdVVek5OTn6woUL9ccff3zC2v7Vr35V/9rXvqbn5ubqRUVF+q9//Wvd7/frn/3sZ3WHw6FXV1frL7zwgq7ruv5f//VfutvtTjrG008/rZ/5ln/22Wf15cuX6xaLRfd4PPrNN9+ceGzq1Kn6P/7jP+p//ud/rjscDr2yslL/1a9+lbT//v379SuvvFLPycnR8/Pz9S984QtjXrOJcOedd+pA4jZ16lT9iiuu0P/P//k/+te+9jXd4/Ho69at03Vd1wH96aefzngbz9eLL76or127Vne73Xp+fr7+oQ99SK+rq0s8vm3bNn3x4sW6xWLRly1bpj/11FNJn8PzObfp+Pxks/Pt+073z//8z7qiKPrevXuTtsfjcX358uX63LlzdU3TdF3XdbfbrT/88MPnbMf5PO+FkunKC7Rz50527drFqlWrANi7dy+HDx/mnnvuQVWTX85FixaxYcMGfv/73wNQUlJCLBbj6aefHjPdmSmPPPIIdrudbdu28YMf/IDvfve7vPrqqwCoqsq///u/c+jQIR555BFef/11vvWtbyX23bt3L+vXr2fu3Lm8++67bN68mZtuuol4PA7AAw88wG9/+1t++ctfcujQIe6++24+/elPs2nTpglre0FBAdu3b+euu+7iy1/+MrfffjuXXHIJu3fv5pprruGOO+4YM0U8nueff55bbrmFG264gT179rBx40ZWrlyZFPPggw+yfPly9uzZw1e+8hW+/OUvc+zYMQACgQDXXnsteXl57Nixg8cff5zXXnuNr371qxOS7+l+8pOf8N3vfpeKigo6OjrYsWMHMPKamM1mtmzZwi9/+csx+2WyjecrEAhwzz33sHPnTjZu3Iiqqtxyyy1omobf7+fGG29k7ty57Nq1i/vvv5+//uu/nrS2ivH97ne/4+qrr2bRokVJ21VV5e677+bw4cPs27cPGOn7XnjhBXw+X+YbOmHlMguM923GarXqdrtdN5lMOqB/8YtfTMT84Q9/OOs3j7/6q7/SrVZr4v63v/1t3Wg06vn5+fp1112n/+AHP9A7OzvH7Jeukdyll16atG3FihX63/zN36SMf/zxx3WPx5O4/4lPfEJfu3ZtythQKKTbbDb9nXfeSdr+uc99Tv/EJz7xPls+tu2xWEy32+36HXfckdjW0dGhA/q77757Xt/216xZo3/qU58a9zmnTp2qf/rTn07c1zRNLyoq0n/xi1/ouq7rv/71r/W8vDzd7/cnYp5//nldVdWU5/T9+tGPfqRPnTo1cf+KK67QlyxZMiaO00ZymW7je9HT06MD+oEDB/Rf/epXusfj0YPBYOLxX/ziFzKSS7Nz9X2n307JyclJ2ud0u3fv1gH90Ucf1XVd1zdt2qRXVFToJpNJX758uf71r39d37x585j9ZCQ3SR599FH27t3Lvn37eOyxx/jjH//I3/7t3ybF6Oc5MvvHf/xHOjs7+eUvf8m8efP45S9/yezZszlw4EA6mj7GwoULk+6XlpYmFr289tprrF+/nvLycpxOJ3fccQd9fX2JkdGpkVwqdXV1DA8Pc/XVVyeuhzgcDn77299SX18/4W03GAx4PB4WLFiQ2FZcXAyQyOdczpZPqudUFIWSkpLE8Y8cOcKiRYuw2+2JmLVr16JpWmK0l27Lli076+MfhDaeqba2lk984hNMnz4dl8tFVVUVAM3NzRw5coSFCxeSk5OTiF+zZs2ktFOM9n2n3053vv3e5ZdfzokTJ9i4cSO33XYbhw4d4rLLLuN73/teGlqdTIrceaisrKSmpoY5c+Zw++238/Wvf50HH3yQUCjEzJkzgZHOJJUjR44kYk7xeDzcfvvt/PCHP+TIkSOUlZXxwx/+MO15AJhMpqT7iqKgaRqNjY3ceOONLFy4kCeffJJdu3bxf//v/wVGFjQAWK3WcY/r9/uBkSnA0z8Qhw8f5oknnkhb20/fpigKAJqmoarqmA9gNBpNun+2fM72nJqmXVC70+n04nWxuOmmm+jv7+ehhx5i27ZtbNu2DRh9n53L+ZxbMTFO9X2n306ZOXPmWfu9UzGnmEwmLrvsMv7mb/6GV155he9+97t873vfO+/z/l5JkXsPDAYDsViMSCTC4sWLmT17Nj/60Y/GdH779u3jtdde4xOf+MS4xzKbzVRXVydWV06WXbt2oWkaDz74IKtXr2bmzJm0t7cnxSxcuHDcZdlz587FYrHQ3Nw85kNRWVmZiRSSFBYW4vP5kl7XM7+Fni2f8zFnzhz27duX9BxbtmxBVVVmzZr1no87kT5obezr6+PYsWP8f//f/8f69euZM2cOAwMDSe3dv38/oVAose30JetwfudWpN/HP/5xXnvttcR1t1M0TeNHP/oRc+fOHXO97nRz584lFoslnet0kCJ3Hvr6+ujs7KS1tZUXX3yRn/zkJ1x55ZW4XC4UReE3v/kNhw8f5tZbb2X79u00Nzfz+OOPc9NNN7FmzZrED7Cfe+45Pv3pT/Pcc89x/Phxjh07xg9/+ENeeOEFPvKRj0xqjjU1NUSjUX76059y4sQJ/vu//3vMQoZ7772XHTt28JWvfIX9+/dz9OhRfvGLX9Db24vT6eSv//qvufvuu3nkkUeor69n9+7d/PSnP+WRRx7JeD6rVq3CZrPx7W9/m/r6en73u9/x8MMPJ8Xcd999/P73v+e+++7jyJEjHDhwgH/5l3857+f41Kc+RU5ODnfeeScHDx7kjTfe4K677uKOO+5ITJ1Otg9aG/Py8vB4PPz617+mrq6O119/nXvuuSfx+Cc/+UkUReELX/gChw8f5oUXXhgzy3E+5/Zi8LOf/eyc0+WT7VTfd/rtVFG6++67WblyJTfddBOPP/44zc3N7Nixg1tvvZUjR47wm9/8JjG7sm7dOn71q1+xa9cuGhsbeeGFF/j2t7+d6EfTSYrcediwYQOlpaVUVVXxxS9+kRtuuIFHH3008fgll1zC1q1bMRgMXH/99dTU1HDvvfdy55138uqrr2KxWICRby42m41vfOMbLF68mNWrV/PYY4/xH//xH9xxxx2TlR4wshL03/7t3/iXf/kX5s+fz//7f/+PBx54IClm5syZvPLKK+zbt4+VK1eyZs0a/vjHP2I0jvzc8nvf+x5///d/zwMPPMCcOXO47rrreP7555k2bVrG88nPz+d//ud/eOGFF1iwYAG///3vuf/++5Ni1q1bx+OPP86zzz7L4sWLueqqqy7ox6g2m42XX36Z/v5+VqxYwW233cb69ev52c9+NsHZvHcftDaqqsof/vAHdu3axfz587n77rv513/918TjDoeDP/3pTxw4cIAlS5bwd3/3d2O+eJzPub0Y9Pb2Ttj16nQ51fedfjv113RycnJ4/fXX+cxnPsO3v/1tampquO666zAYDGzdujXxGzmAa6+9lkceeYRrrrmGOXPmcNddd3Httdfy2GOPpT0HRT/fK4f/C6xbt47Fixd/IP6U0/33388zzzwj0zDif73GxkamTZvGnj175E91pckHpe9Lx7mWkdwZfv7zn+NwODK22vFMzc3NOBwO/umf/mlSnl8I8b/TZPd9119//Zi/rjIRZCR3mra2NoLBIABTpkzBbDZnvA2xWIzGxkYALBbLpCzaEOKDREZy6fdB6PvS1QYpckIIIbKWTFcKIYTIWlLkhBBCZC0pckIIIbKWFLk0CIfD3H///YTD4cluyoSRnC4OktPFIRtzgg9mXrLwJA28Xi9ut5uhoaG0/5o/UySni4PkdHHIxpzgg5mXjOSEEEJkLSlyQgghspZxshuQKZqm0d7ejtPpTPzR0HTxer1J/80GktPFQXK6OGRjTpC5vHRdx+fzUVZWhqqefaz2v+aaXGtrq/z1ECGEyCItLS1UVFScNeZ/zUjO6XQCcIX7YxiVzP/JmnTpvmX2ZDdhwrlPpPd/ojgZQh7TuYMuMtbe7DtPAMZA9v0PWA39/sluwoSKaRHebPpVol8/m/81Re7UFKVRMWdVkTOYcya7CRPOaMy+S8VGU/YVuWw8TwBGg2GymzDhDGr2FW7gvC49Zee7VAghhECKnBBCiCwmRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrGWc7AZcDJpDh2kIHyCiBXEa8pltW0OusXDc+M5IA3XBXQQ1PzbVxUzbCgpNlSljDwW20Bo5yizrKqpy5qcrhTF6Dm6me9+bxII+rJ4yytfegr1oyrjxg/X76Nj5IhHfABZ3AWWrbsQ1ZQ4AejxOx44X8bYcIeLtRzXn4CyfQdmqD2GyuzOVEgBtLe/S3PwWkYgfh6OEGTM/jMud+rUH6O46QMOJVwmFBrBZPUyvuQ5PwezE4w0nXqO7az/h0CCqasDhLGd69TW43OO/VhOts3YL7UffJBryYcstZdrSW3B4xn/+vpZ9tBx4iXBggBxnAVMWfoi8sjmJx7c++tcp95uy6EOUzb5ywtufSmvru7Q0v00k4sfuKGHmzJtwuc5ynrpPnadBrFYP1dXX4SmYlXi84cRrdHfvJxQaQlUNOJ3lTJt+De6znPuJ1tK1jcaOd4hE/ThsxcyeegNuR8W48V39h6hrfZ1QeBBbTj41lVdTmDsTAE2LU9+2kd7BWobDAxgNOXhc06mp3ECO2ZWplGga2kPD4A4i8QBOcyFzCtaTm1M6bnyn/xi1/VsIxoawmfKYlX85hfbpicd1XaduYAut3gNEtTB5OWXMLbgauzkvbTlcFCO5devW8fWvf31SnrsjcoKjwW3U5CxhjesjOA357PK/RFgLpowfiHWxP/AG5ZaZrHHdTJF5Knv8r+GL94+J7Yo0MhTvxqLY0p1Gchvr9tD+7rOULLuGWbfejTW/jBPP/5po0JcyPtDZQOPG/8EzaxWzbr0Hd9V8Gl7+L4L9HQBosQjDva0UL72ambfezbRrPkt4qIcTL/1nJtOiu2s/dbXPUzVtPctXfBWHo5T9e/+TSMSfMn5osInDh/5Aadlylq+8i4LCuRzc/z/4/Z2JGJutgBmzPsyK1V9nybIvkWPNY9+e8Y850Xqb99K091kq5l3Ngmu+jj23jCObHiIaSn2ufL2N1L77/yiavpKF195Nfvl8jm95mOHBjkTM0g//Q9Jt+oo/AxTyKxZmJKeurv3U1b5AVdV6lq/4Pzgcpezb+1/jn6ehJg4fepTS0uUsX/FVCgrncuBAivM088OsXPU1li79S3Jy8th3lnM/0Tr7DnKs+WWml69j1fy/xGkrYfex/yYSTf38g75mDtQ9QXnhElbN/xKFebPZV/sH/MNdAMS1KN5AB9PKrmD1vC+xaMbHCIR62Xv89xnJB6DDf5SjvW9Sk7eGSyruwGkuYmfHE4RjgZTxA6E29nU9R4VzPpdUfIZiew27O5/BF+5JxDQMbqdpaA9zC69mTfmnMCgmdnY8QVyLpS2Pi6LIPfXUU3zve98DoKqqih//+McZe+6m0EEqLLMot8zEYchjrm0tBoy0RY6njG8OHaLAVMG0nIU4DLnMsC7DZfDQHDqSFBfSAhwZfpeF9nUoSmZPQ8+Bt/DMWY1n9kpy8kqouPxWVKOJ/qPbx4l/G1flLIoWX0lOXjGlK67HWlBO78EtABgsVmpu/BJ51YvJyS3CXjyVirW3EOxtJeIbyFheLc1vU1q+gtKy5dgdxcycfTOqwUxH+86U8a0tW8jPn8GUqZdjtxcxrfoaHM4y2lrfTcQUlywmP78GqzUfu6OYmhkfIh4PEzitg02njmObKJq+iqLpK7G5S5i2fORcdTfsSB1//G1yS2ZRNvtKrK5iKhdchz23nM66LYkYs9WVdBtoP4SrqJochycjObW0bKasbAWlZcuw24uZNesjqKqZjvZdKeNbW95JOk/Tp1+N01lGW+vWRMzY83QD8Xg4qRCmU1PnO1QULqO8cAkOaxFzqm7EoJpo69mTMr65aysedw1VpZfisBZSU7Eel62U5q6Rz6DJmMOy2XdS4pmP3VpArqOS2VM/hG+4nWB4MCM5NQ7upNK1gArXAhzmAuYVXo1BMdHmO5gyvmlwNwW2aUzLW4nD7GFG/qW4LMU0e/cCI6O4pqHdVOetptheg9NSyIKiGwjH/XQH6tKWx0VR5PLz83E6nRl/Xk2P44334jGWJbYpioLHVMZgrDvlPoOxbvJPiwcoMFUwGB+N13WdA4FNTMtZgMOQvmF6Klo8xnBPK47yGYltiqLiqJhJoKsp5T6B7iYc5TOTtjkrZhHoahz3eeKREKBgsFgnotnnpGkxfL528vJrEtsURSUvrxrvUHPKfbxDzUnxAPmeGePGa1qM9rbtGIw52B3jT9lMFC0eIzDQhrt49LVXFBV38Qz8vanPlb+vCXfxjKRt7tJZ48ZHQj4G249QNH3lxDX8LDQthj/FecrPr8brTf26D6U6T/kzGBonXtNitLfvwGjMwZGJ86TF8AU6yHePTsspikq+azpD/paU+wz5W5PiATzu6nHjAWLxkc+UyZgzIe0+G02P4w134bFNTWxTFAWPdQqDofaU+wyG2/FYpyZtK7BVJeKDsSHC8UBSjMlgwW0pZTCc+pgT4aK4Jrdu3ToWL17M3r17aWpq4u677+buu+8GRgpGukT0EDo6FjW5ozYrVgLxoZT7hPVgivgcItpw4n5DaD8KClMs8ya+0ecQDwVA1zBZk780mKwOwoOpC3ds2IfJ5kiOtzmJjTO9qcWitG97nryaxRjM6f9AAkSjw6BrmM3J7TSbnQwP96TcJxLxp4h3EAknTzH19h7h8ME/oMWjmC1OFi35C8xm+8QmkEIscvJc5Zzx2uc4CXpTn6toyIcp54xza3GMO73Z27AT1WQhv2LBxDT6HKLRYfQU58lkdhA423kypTpPyTn19h7l8KE/EI9HMZudLFqcmfMUiQ2jo2E2ntFGk4NAqDflPuFoipxMjnGnN+NalNqWVynxzMdoSP9nKhIPoqNjNiS/fhajnUBw7KUXgHAsgNmQfOnFYrARjo9Mb576b8qYcaZAJ8JFMZI75amnnqKiooLvfve7dHR00NHRMW5sOBzG6/Um3T4IhmK9NIUPMd9+OYqiTHZzJpwej9P42m8BnYrLbpvs5kyIvLxqlq+8iyXLv0R+/kwOH/h9xq71pFt3w3YKpixFNZgmuynvW17edJavuIuly/4Sj2cGhw5mx3nStDj76x4HYE7VjZPcmovPRVXk8vPzMRgMOJ1OSkpKKCkpGTf2gQcewO12J26VlRe+ysqs5KCgjFlkEtGDmNXU03AWxZoiPoRZHfn2MhDrJKIHeWvoUV4Z+E9eGfhPQpqfY8HtbBp69ILbeKEMOXZQ1DGLTKJBP0Zr6ilho81JdDi5s4gO+8bEnypwEd8A1R/6y4yN4gBMJhso6phOLRLxYTanzstsdqSI92O2JH/DNhjM2GwFuN1TmD33VhRFHfc630Qymk+eq9AZr33Ihzkn9Qo7U45zzKgtGvaPGd0BeHtOEPL1UDR91cQ1+hxMJhtKivMUjfixnO08RVOdp+T4kfPkGTlPczJ3nsxGGwoqkdgZbYz6sZwxWjvFkmLUFkkxutO0OPvrHyMUHmTprM9kZBQHYDZYUVCIxJNHWOFYAIsh9ejYYrQTiQ8nx8eHE/Gn/psyxpi+EfdFVeQuxL333svQ0FDi1tIy/lz3eFTFgMtQQH9sdMSo6zp90XZyjUUp98k1FtEfS55f7ou2kWsYiS8z13CJ6xbWuG5O3CyKjWmWBSx3XHvBbbxQqsGIrbACf1ttYpuua/jbarEXT025j71oalI8gK/tOPbiqtFjnCxw4aFeam78Esac9E8TnU5VjTidZQz214+2SdcYGKgfd7m/yz2FwYH6pG0D/XXn/HmAjo6WxtVgp6gGI/a8coa6ks+Vt6sOR0Hqc+XwTGWoO/lcDXUeTxnffWI79rwK7HllYx5LF1U14nCWMTAwutAgcZ5cqV93t3sKA/3J56m/vw73OPGjx83QeVKNOO2l9A+dOO25Nfq9Dbgdqb9cux0V9HtPJG3r855Iij9V4IZD/SybfSdmU+ZWYauKAZelmL7h0eueuq7TF2wmNyf1+yXXUkZfMPnab99wUyLeanRjMdiTYmJamKFwB7mW9L0Hs7bIWSwWXC5X0u29mJozn9bwMdrCtfjjgxwe3kKcGOXmkcUABwKbOB4cXek2JWcevdFWGkMH8McHqQvuZijey5Sckd8pmdUcnIb8pJuiqJhVK3ZD7vvO+3wULricvqPb6D+2g9BAF61vP4kWjZA/a2TxQdPrv6N92/OnxV+Gt/Uo3fveJDTQRcfOlwn2tFIwfy0wUuAaXn2E4Z4Wpq7/FLquER32Eh32osXT38mcUjnlMtrbd9DZsYtAoJvjR/+IFo9QWroMgCOHHuNE3UuJ+IrKtfT3Hael6W0CgW4aTryGz9tGecUaAOLxCCfqXmZoqJlQcACft42jh58gHPZSVJSZa1ils66g+8Q2ehp2EPR20bDzKeKxCIXTVgBQt/X3NO9/YTR+5mUMdRyj/eibBL3dtBx8mcBAKyU1a5OOG4uG6G/Zl7EFJ6errLyUjvaddHTsHjlPx/5IPB6htGwpAIcPP059/cuJ+IrKS+jvP05z82nnyddGecVqYOQ81dcnn6cjR54kEsnceZpacgltPbtp79mLP9jDkcbniGsRygqXAHCw/ilqW15NxE8pXk3fUB2NHVsIBHuob30Db6CdKcUj52NkivJRvIF2FlTfiq5rhCM+whFfRgo3QFXuclp9+2nzHsQf6eNQ76vE9SjlzpHf8+7veoFjfW+Nvga5S+kdbqRhcAf+SB+1/VsYCncyxbUYGFm4MtW9lPqBrXQH6vCFe9jf9SIWg4Mie02qJkyIi2LhyenMZjPxeDxjz1dqnk5EC1EX2kVYC+IyeFjmuDaxuCSo+YHRa2t5xmIW2q+kNriL48Gd2FUXSxwbcBryM9bmc8mrWUIsFKBj58vEhr1YC8qZfsMXMNlGpn8i/kE47XqhvWQaVVd9mo4dL9Kx/QUs7kKmXfvnWPNHVq5FhofwNh0C4NgTDyY9V/VNX8ZZlr438OmKihcSifhpOPEakbAPh7OUhYv/PDGtFQol5+XOncqceR+n4cQrnKh/GautgPkLP43DcWoaXGF4uIfOA7uJRgKYTDacrgqWLPsidkdxRnIqmLKYWNhPy8GXT/4YvIzZV3we88npx/DwQFJOzoIqatZ8ipYDL9Fy4EVyHAXMXPtZbLnJqwz7mvcC4JmyJCN5nK64eCHRaGDkPEVOnqdFf56YVg6HBlFO+0y53VOZO+9jnDjxKifqX8Fm87BgwdjzdPDAHqLRkfPkclWwZGnmzlOJZz6RWID6ttcJR/04bSUsnXVHYroyFBlKOk+5ziksqL6NutaN1LVuxJbjYdGMj+OwjbQ3HPXSM3gMgK0Hf5H0XMtmf5Z817S051TqmE0kPkztwBbCsWFclkKWl96WmFoMxrxJOeXllLOo+EMc79/M8b7N2E25LC25Gadl9A9nTMtdSVyPcrDnFWJamLyccpaX3opBTV8pUvR0Lk+cIKdWV/74xz/mmmuuwWq18vOf/xyLxUJBQcF5HcPr9eJ2u1mfewdGxZzmFmdO15/NnewmTLjcushkN2HChQou/oUdZ7L2ZN95AjD6o5PdhAln6Eu9uvZiFdPCvNbwU4aGhs45S3fRTVd+97vfpbGxkerqagoLx//TWkIIIcRFMV355ptvJv69evVq9u3bN3mNEUIIcdG46EZyQgghxPmSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1jJPdgEzTAiE0JT7ZzZgw3prJbsHEK9wdnuwmTLjO1ZbJbsKEc56ITHYT0kJt6prsJky4uNc72U2YUHE9et6xMpITQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWcbIbcDFoiR+nMX6ECEEcSh6zDctwqwUpY1vjdXRoDfj1QQBcSj41hkVJ8Qdj79KhNSTt51FKWWq6Mm05nGloy2aGNr1J3OfDXFqG5+ZbyJkyJWVs4MB+Bl7fSKy3Fz2uYSoowH3FFTiXLU+Ki3R10f/CcwRPnIC4hrm4mOLP3IkxLy8TKdHStZ3Gzi1Eon4cthJmT7ket6Ni3Piu/kPUtb1OKDyILcdDTcUGCnNnJsX4gz3Utr7KoK8JTddw5BSysObPsFpy05zNqIGdm+nb+gZxvw9LcRnF19yCtXzquPHeI3vp3fQS0cF+zPkFFF51I46auYnHdV2n962XGNyzFS0cxFoxjZLrb8OcX5iJdICJP1dd/Ydp7dmJL9BBNB5k9by/xGkrzUQqCc3DB2kY3ktEG8Zp9DDbeSm5puJx4ztD9dQFthOM+7AZ3Mx0rKbQMnpeu0InaAkewhvrIaqHWZN3Oy5T6n4nXVpix2iMndb3mZaP2/cBdMWbqIvtJ6T7sSlOaoxLKDSUJx7XdZ362H7a4nXEiJKrFjLbuAK76kpbDjKSO4fOeBPH4ruZbpjPKtP1OJVcdsfeIKKHUsYP6F2UqFNZbtzAStM15Ch2dsfeIKQPJ8V5lFIuN92SuC0wrs1EOgD49+6h70/Pknf1NZR//W7MZWV0/sevift9KeNVm428qzZQ9tW/ouKeb+BcsYKexx5l+NjRREy0t5f2n/8MU2ERZV/6MhX3fIPcDRtQTJn5HtXZd5BjLS8zvWwdq+b9JU5bMbuP/w+RqD9l/KCvmQP1T1BesJRV875EYe5s9tX9Af9wVyJmONTPziP/iT2ngGWzPsuaeV9mWtnlGNTMfTf0Ht5D92t/pOCya6n63D1Yispo+cOviQVSn6vh1gban/4f3ItWUvX5b+CYuYDWx/+LcHdHIqb/3dcZ2PE2JdffztTPfh3VZKbl979Ci0UzklM6zlVci5LrmEJN5YaM5HCmjlAdR/1bqLEvZ03+bTiNHnYNPkdYG04ZPxDtZL/3VcpzZrMm/3aKLNPYM/QSvlhfIiauR8k1lzLTsTpTaSTpjDdyLLab6cYFrDLfgFPNY3dk/L5vUOvhQHQL5YZqVplvoFCtZF/0LfzaYCKmMX6Ylvgx5phWstJ8LQaM7Im+QVyPpy0PKXLn0KQdpUKtptxQjUNxM8ewEgNG2rT6lPELjGupNMzEqeZhV9zMNaxER6df60yKUzFgUayJm0kxZyIdAIbeegvXqtU4V6zEXFxCwUdvRTGZ8G3fnjLeWl2DfcECzMXFI6O4yy7HXFpKqGF0NNr/0ovYZs/Bc+NNWMorMBUUYJ83H4PDmZGcmrrepaJwKeWFS3BYi5gz9UYMqom23j0p45u7tuFx11BVuhaHtZCaiqtw2Upp7h59DeraNlKQO4OZldfgspdiy8mnKG82ZpMjIzkB9G/bhHvxanIXrcRSWELJDbehGk0M7Ut9rga2v429ejaeNVdhKSimcN315JSUM7BzMzDyTbp/+1t4Lr0a56z55BSXUfrhTxLzefEfO5iRnNJxrsoKFlFdvg6Pa3pGcjhT0/A+KqxzKbfOxmHMZ67zCgyKibbg0ZTxzcP7KTBPYZp9CQ5jHjMcK3EZC2geHj0HZdZZ1NiX4zGPP8JNp6bYUSoMNZQbq3GobuYYV2LAQFs8dd/XHDuKRy2lyjgXh+qmxrQIl5JHc/wYMPLea44dZZpxPkWGSpxqHvNMawjrw/RoLWnLQ4rcWWh6HJ/eT75aktimKAr5aglDWu95HSNOHB0dk2JJ2j6gd/Fm5Em2RP7Ekdh2Inp4Qts+Hj0WI9zWinXGjMQ2RVWxzphJqKnp3PvrOsHa40S7e8iZPtKh6JrG8NEjmAoK6XjoVzTefx9t//4TAgcPpC2P02laDF+gnfzTOjhFUcl3TWfI35pyn6FAS1I8gMddk4jXdY3ewVpsOR52H/tv3tzzA7YdfojugSPpS+QMejxGqKMV+7TRaTlFUbFNm0mwtTHlPsG2RuzTZiRts0+fTbBtJD462E884MNeNXpMQ46VnPIpiZh0Sse5mmyaHscb60kqRoqi4DGXMxjtSrnPYLSLfHN50rYCcyWDsdTxmfZe+r4hrZd8NXmK2KOWJeKDup8IITynHdOkmHEpBQyeZ3/6Xlx0Re6JJ55gwYIFWK1WPB4PGzZsIBAIpOW5IoTR0TGTk7TdTA5hUg/Zz1Qb34sFK/nK6IktUEqZZ1zDMuN6ZhgXM6B1syf2BrquTWj7U4kHAqBpY0ZYBoeDuC/1FBiAFgzS8Hf30vC336LzP3+D5+absc2cNXJMvx89HGbwjdexzppN6Re+iH3+fLp++wjB+tTf+iZSJDY8cp7OGGGZTXbC40yBhaP+lPGnpswisQBxLUJDx2Y87hqWzbqDorzZ7Kt7lH5vY1ryOFNsOAC6htGefK6Mdue405Uxv++s8bGAN7FtTMw409UTKR3narJFtBA6OhbVmrTdrNqIjDNdGdaGsai2sfHx1PGZluj7lDP6PiWHsB5MuU+YUMr4U9ObkZN9pllJfp0sSg6RcY45ES6qhScdHR184hOf4Ac/+AG33HILPp+Pt99+G13Xx8SGw2HC4dHRkdfrzWRTAWiIH6JTa2K5cT0GxZDYXmKoSvzbSS4OUx5bos/Sr3fjOa0YfpAoFgsVd38DLRwmWFdL/5+exeTxYK2ugZOvv23ePHIvvwIAS3k5oaZGvFvfwVpdPZlNf09OvaeKcmcxtWQNAE5bKYP+Flp7dpLvqprE1gkhztdFV+RisRgf/ehHmTp1ZBXSggULUsY+8MADfOc733lfz2fGgoKS+AZySoQQljNGd2dqjB+hMX6YpcarcKpnX11oUxyYsBDUfUB6i5zBbgdVHbPIJO73Y3COf/1MUVVMBSOrqizl5US7uxh8fSPW6prEMc3FySvJTEXFSdft0sVstI2cpzO+2UeiASzjXD+zmBwp40+NGMxGG4qiYrcmrzh05BQy4G+ewNaPz2izg6KOGbXFAmNHa4l9HGNHeafHG+2u0W1OV1JMTnHy9Fk6pONcTTazmoOCQlhLHo1EtGHMZ4zWTrGotjGLUiLaMGZD6vhMS/R9ZywyieghLGeMxE6xkJMy/tTo7tSMWEQPJh0jrIfO2Ue+HxfVdOWiRYtYv349CxYs4Pbbb+ehhx5iYGAgZey9997L0NBQ4tbScuEXNlXFgFPJp18bnSfX9ZFFJGdbRtsYP0xD/CBLjFfiVj3nfJ6QPkyU8JhhfDooRiOW8gqCdbWJbbqmEayrJWfq+MvSz6TrOnosPnrMykqiPT1JMdGenoz8fEBVjTjtZfR7Rwuqrmv0e0+Muyzdba9MigfoG6pPxKuqEZetjOFQX1JMINSH1eye4AxSUwxGckorCDSedq50jeHGWqwVVSn3sZZXEWioTdo23HAca/lIvCk3H4PdmXTMeDhEqK05EZNO6ThXk01VDLiMhfRHRq8R6rpOX6Rt3J8Q5JqK6Y+0JW3ri7SSaxz/JweZNNr3jS6YO1ff51YLxiyw69M6EvFWxYGZHPpO609jehSv3kvuWfrT9+uiKnIGg4FXX32VF198kblz5/LTn/6UWbNm0ZBitGCxWHC5XEm392KqOps2rY72+An8+hBH4juIE6NMHbkQfjD2DrWxvYn4hvhh6uL7mWtchVWxE9aDhPUgMX1keXZMj3I8todBrZeg7qdP62RvbBM2nBQomfldj/vyy/Ft24Zv5w4iXV30PvUkeiSCY8VKALp//zv6X3g+ET/w+kaGjx8j2tdHpKuLwU1v4t+1C8fSpYmY3CuuxL9vL95tW4n29jK0ZTPDRw7jvuSSjOQ0tXgNbT27aO/diz/Yw5Gm54lrUcoKlgBw8MRT1La8loifUryKPm8djZ3vEAj2UN/2Bt7hdqYUrUzEVJWupbP/IK09uxgO9dHctY3ewWNUFK3ISE4A+auuYGjPVob27yDc20XXi0+gRSO4F460s/3Z39H9xnOJ+LyVlxE4cZS+rW8S7u2i562XCHa0kLf8UuDk4oGVl9O35VV8xw8S6m6n49nfYXS6cMyan5Gc0nGuorFhfMMd+IMjX7QCwT58wx2Eo+m/zggw1baI1uAR2oJH8ccGOOx7i7gepdw6G4AD3o0c928dzcm2kN5IC43De/HHBqjz72Ao1sMU2+g5iGghvNFe/LGRL/KB+CDeaC/hDF23m2qcTVv8ZN+nDXEktp04ccoMJ/u+yDvURkdXxE4xzqZPa6cxdoSANkR9dD9evZ8phpFr94qiMMU4m4bYQbrjrfi0AQ5G38Gi2ChUK9OWx0U1XQkjL9TatWtZu3Yt//AP/8DUqVN5+umnueeee9LyfCWGqUQIUR/fTzgewqnksdR4ZWK4HdKHQVES8a3xWnQ09sc2Jx1nujqfauNCFBT8+gDtsRPEiGLBikctodq4EPW063bp5Fi8hHggwMDLLxPzebGUlVPy+S9gPDldGRscTMpJj0Toffop4oODKCYTpqIiij7xSRyLlyRi7AsWUPDRWxl843X6nnkaU2ERxXfcSc60zCzpLvHMJxILUN/2BuGoH6ethKUzP52YAgtFhoDRnHKdU1gw/Vbq2l6nrnUjtpx8FtV8HIdt9Jt0Ud4c5ky9kYaOzRxrehFbjoeFNR8jz3n+I973yzV3CfGAn55NLxEPeLEUl1P58S9iPLlwKDo0kHSubBXTKLv50/S++SK9bz6PKb+Qitv/HEvR6Beo/DVXoUUjdL7wOFooiLVyGpUf/yKq0ZSRnNJxrnoGj3Go4Y+J+wdOPAHA9LIrqC5P/x9ZKM2pIaIFqQvsIKwN4zIWsCz3xsTikmDcn5RTnqmEha4N1Aa2cdy/DbvBzRL3dTiNozM/PeFGDvreSNzf730VgGrbcmoc6f+iVWKoIqKHqY/uI8zJvs98et8XSMopVy1kgWktdbF91MX2YlOcLDJdjkPNTcRUGeYS12MciW4jRoRctYglpiuT1ixMNEVPtWrjA2rbtm1s3LiRa665hqKiIrZt28anP/1pnnnmGa6//vqz7uv1enG73Vxpuh2jkpkPcybU/+OyyW7ChKt+LDPfvjOp5Zr0/UWHyVL5SuYXc2WC2vTBWMY/kbRJWHiXTjE9yhvhxxgaGjrnLN1FNZJzuVy89dZb/PjHP8br9TJ16lQefPDBcxY4IYQQ/ztdVEVuzpw5vPTSS5PdDCGEEBeJi2rhiRBCCHEhpMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLeNkNyDTFLMRRTFNdjMmzMwfNUx2Eybc4b+fMtlNmHDmPn2ymzDhdGOWfkcOhye7BRNOUZTJbsKEupBssvRdKoQQQkiRE0IIkcWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylnGyG3AxaI4cpTF6kIgexKHmM8eyErehMGWsPz5AXWQvXq2PkB5glnkFU81zk2Lqwns5Ed2XtM2muLjUfkvacjhTU2A/Df49ROLDOE0FzHFfTq65OGWsL9pHnW8bQ9EeQnEfs12XUuVYnBRT79tJV+gEgdgABsVIrrmEma5LcBjzMpDNaW198x28r24i7vVhrigl72MfwVI15Zz7BXbspe8/f4d10TwKv3RnYruu6ww99wr+zdvRg0HM06vI/+QtmIpSn/90GNy6mcHNbxL3+zCXlFF44y3kVKTOKdzVSf/Glwi3txIbHKDgho+Qe8nlSTGNP/w+scGBMfu6V11C4U23piWHM7V0bqOpfQuRiB+HvZhZVR/C7awYN76r7yD1za8TCg9izclnxtRrKMibmTL2yIlnaevaycyq65hSekm6UhijOXyYhvBIP+E05DE7Zw25xvHfJ53RBupCuwlqfmyqi5k5yyk0VSYe74o20hI5ijfeR1QPs8bxEVwGTyZSSWiOHaMxdnik71PymGNegVstGDe+M95EXXQfId2PTXExw7SEQkN54vGueDOtsVq8Wh9RIqy23IBLzU9rDhkfya1bt46vf/3rmX7a96wz2sCxyA6qzYtYbbsJp5rHruBrhLVgyvg4cayqkxnmZZgV67jHtau5XGH7s8Rtpe36dKUwRkewlqNDm6lxruCSwo/hNHnY2fcs4fhwynhNj2E1uJnlWoNFtaWMGYi0M8W+gNUFt7Hc8xF0XWNn37PEtGg6U0kS2LmXgSf/hPtDGyj99tcwVZTS/e+/Ie71n3W/WF8/g089j6Vm2pjHfK+8ie+NLeR/8qMUf+suVIuZ7n//DXo0M3n5Duyh98Vnyb/yGiq/cjeWkjLaH/41Mb8vZbwejWDK9+C55kMYHM6UMZVf/jpVf3Nf4lb22b8EwD5vUdryOF1n7wGON77E9Ip1rFz4JZy2EvYc+S2RaOrzNOhr5uDxJygrWsqqhV+mKH8O+479Hv9w15jY7r7DDPlasZhS554uHZETHA1tpyZnMWscH8ap5rMr8PK4/cRArIv9w29Sbp7JGsdHKDJNYc/wRnzx0S8fcT1GrqGYmTnLM5VGks5YI8eiu6g2LmS15YaRvi/8OmE9lDJ+MN7Dgchmyg3VrLZ8iCJDBXsjm/Bpg4mYuB4jVy1khmlJhrKQ6cpzaowepsI0g3LTDBxqLnMtazAoBtpjdSnj3YYCZlmWU2qahnqWl1dFwaJaEzezkpOuFMZo9O+l0jaPCttcHKZ85rmvxKAYaRs+kjLebS5mtnstpdaZKIohZcxyz4epsM3BafLgMhWwIHcDobgPb7Q7nakk8W18G8faVTguWYGptJj8T3wU1WzC/+6OcffRNY3e//w97huvxliQ/I1S13W8r2/Gff16bIvmYa4oxfPZjxEf8jK891C60wFgcMtbuJevxrVsJeaiEgo/fCuKyYRv1/aU8TkVUyi47iacC5egGFNP1BjsDoxOV+IWOHYYU74H67TqdKaS0NzxDuVFyygrWorDVsTs6TdhUE20d+9OGd/SsRVPbg1V5ZditxVSPWU9TnspLZ3bkuJCYS/HGl9g/ozbUNTU79N0aYocpMI8i3LzTByGPOZa1458piLHU8Y3Rw5TYKxgmmUBDkMuM3KW4TJ4aI4cTsSUmWuoyVmCx1iWqTSSNMaOUGGoodxYPdL3mVZhYPy+ryl+FI9axjTTPByqmxrTYlxKPi2xY4mYMuN0qk0L8ailmUojs0Xus5/9LJs2beInP/kJiqKgKAqNjY1s2rSJlStXYrFYKC0t5W//9m+JxWKJ/cLhMH/1V39FUVEROTk5XHrppezYMX7HNVE0PY5P68NjGH2TKYpCvqGMwXjP+zp2QPOxKfAYbweeZH/oLYLa2UcbE0XT43ij3Xgso9MiiqLgsVQwGO2csOeJ6mEATGpmirceixFpbiNndk1im6Kq5MyeQeRE07j7DT3/GganA8falWMei/f2o3l95MyekdimWq1YplUSbhj/mBNFj8UIt7dirR59fkVVsVXPJNQyMc+vx2L49u3CuXQliqJMyDHPRtNi+Pwd5OeOFlRFUcnPrWbQ15pyn0FfC/m505O2eXJrGPK1JO7rusahuieZWrYWh60oPY0fh6bH8cb7koqRoih4jOP3E4OxbvLPKF4FxnIGY5n7Ung2mh7Hp/fjMYwWo5G+r5RBrTflPkNaDx5DSdI2j6GUQe399ZXvV0aL3E9+8hPWrFnDF77wBTo6Oujo6MBkMnHDDTewYsUK9u3bxy9+8Qt+85vf8P3vfz+x37e+9S2efPJJHnnkEXbv3k1NTQ3XXnst/f394z5XOBzG6/Um3S5URA+jo48ZZVmUHMJ66mmI8+E2FDA/Zy1LczYwx7KaoOZnR/AlYnr6p8AiWnAkJ0PyVKpFtY07XXmhdF3n6NDb5JpLcZoycw0h7g+ApmFwJU9TqS4HcW/qqb1QXQOBd3aQ/+nbUh/z5H4GlyNpu8HpRBvnmBMpPnwypzOmHQ0Ox7jTlRfKf+QgWiiEa+mKCTneuURjw+homE32pO1mk51INHVOkagfs8lxRrwjaXqzsX0ziqJSWbJ64ht9Dqf6CcsZlyfMipWInvozFdaDWM7oV0bi33u/MpEinOz7OP++L6yHxsSblRwi40xvZkpGi5zb7cZsNmOz2SgpKaGkpISf//znVFZW8rOf/YzZs2dz8803853vfIcHH3wQTdMIBAL84he/4F//9V+5/vrrmTt3Lg899BBWq5Xf/OY34z7XAw88gNvtTtwqKyvHjc20QmMFJcYqnIZ8CozlLLVuIKZH6Iw1TnbTJsThoU34Yv0szrt2spsyLi0Uou/hP5D/qVsxOOzn3iFLeXdtwzZjNkaXe7Kb8p55/e20dGxlXs0tGRmNiovLpK+uPHLkCGvWrEl6c65duxa/309rayuDg4NEo1HWrl2beNxkMrFy5UqOHEl9DQng3nvv5Z577knc93q9F1zozIoFBWXMN5GwHhrzre39MClmbKqLoHbho80LZVatIznFk7+NhbVhLIbUi0ouxOHBTfSEGllZ8FFyDI5z7zBBDA47qOqYUZvm9Y8Z3QHEevqJ9w3Q84uHRzfqOgDN/+dvKb3/m4n94l4/BrcrERb3+TBVpP86icF2MqczRm1xvx/jOItKLkR0oJ9gfS0ln/zs+z7W+TIZbSioRKKBpO2RaADzOItFzhy1jcSPju4GfY1EogE27/q3xOM6GscbX6a5YyuXLr2HdDrVT5w5wonoQcxK6s+URbGOWcAxEj9x/cr7YeZk38f5930WJWdMfEQPZXS9QSqTXuTSxWKxYLFY3tcxVMWAU/XQF++gyDiyZFvXdfrjHUwxzZ6IZgIQ06MMaz5Kjem/8K8qBlymIvoiLRRbR65z6LpOX7iVqfaF7/m4uq5zZOgtukInWFlwCzaj69w7TSDFaMQ8pZzQsTpsi+ePtEnTCB2rw7Fu7DJyU0khJf9fcuc39KeX0UJh8m7/MMY8NxgMqC4noWO1mCtHipoWDBFuaMFx2ZqM5GQpqyB4ohbH3AWJnIZP1JK7au059j437+4dGOwO7DPnvO9jnS9VNeJ0lNI/dIKi/JHn1XWN/qETVJaMvS4KkOuspH/oRNLPAfoH63E7R760lhQsJt+d/NnZc/i3lBQuoqxoaZoyGaUqBlwGD/2xdopNU4GTn6lYO1PMqV/bXGMR/bF2qizzEtv6Yu3kGjN7PXE8qmLAqeTTF++kyDDyOo/0fZ1MMab+6YZbLaQv3slU42jOfVoHuWrmfm6TSsZXV5rNZuLxeOL+nDlzePfdd9FPfosG2LJlC06nk4qKCqqrqzGbzWzZsiXxeDQaZceOHcydm/z7s3SoMs2lLXqctmgdfm2QI+GtxPUYZcaRBQ4HQm9TG96ViB+5CN2PN96PjkZIH8Yb72f4tFHasfAO+uOdBDU/g/Fu9obeQEGh1DR2CXtacnIspjVwmLbhI/ij/RwaepO4HqPcNvLm3D/wKse87yTnFO3BG+1B1+OE4gG80R4CscFEzOGhTbQHj7Eo7xqMiolwPEA4HiCux858+rRxrr8M/+bt+N/dSbSji4HfP40WjuBYM7IEu/fhPzD4zIsAKCYT5vKSpJtqzUHNsWAuL0ExGlEUBddVlzL0wusM7ztEpK2DvkcexeB2YVs872xNmTC5ay/Hu3Mb3t07iHR30fPsk+iRCM5lIwWh64nf0fvK84l4PRYj3NFGuKMNPR4n5h0i3NFGpC95sYCuafh278C5ZDmKIbMrEaeUXkJ71y7au/cQGO7h6InniMcjlBaOFKSDtU9S1/RqIr6ydDV9g3U0tW8hEOyhvuV1vIF2KktWAWA22XDYipNuimrAYnZgt47/m66JNNU8n9bIcdoitfjjgxwOvTPymTKPFIQDw5s4Hto5+hqY59Iba6UxfAB/fJC60G6G4r1MOe03tREtjDfehz8+CEAgPoQ33kdYm5hr5+dSZZxDW7yWtlg9fm2II9FtxIlRdvLL+IHIFmqjexLxUw2z6dPaaYweJqANURfdh1frp9I4KxET1cN4tX78+hAAw5oXr9b/vtY4nEvGR3JVVVVs27aNxsZGHA4HX/nKV/jxj3/MXXfdxVe/+lWOHTvGfffdxz333IOqqtjtdr785S/zzW9+k/z8fKZMmcIPfvADhoeH+dznPpf29paYphHRQ9RH9hLWgzjVfJZaN2BRR4bsIS2Aoo5OtYb1IFuDf0rcb4oeoil6iDy1mBW2607GDHMg9BYRPYxZySHPUMQq2w0ZG9aXWmcQ0YLU+rYTjgdwmQpZ7rkpMV0ZjPuA0ZxC8QDv9DyauN8Y2ENjYA955jJWFXwUgJbhgwBs73s66bnm566nwpaZkYJ9+WI0f4Ch5145+WPwMoru+tzotGP/4AVfs3Fesw4tEqH/d0+iDYewVFdRdNfnUEymdKQw9vkXLCEeCNC/8WVifi+W0nLK7vxCYroyOjgIp+UU83lp+b+j03aDm99kcPOb5FRVU/H5ryS2B+triQ0N4Fq2KiN5nK6kYAHR6DAnWl4nHPXjtJewZM4dWMwj04+hyFDSecp1TmH+jNuob95IXfNr2HI8LJr1CRy21H+8YDKUmqcT0UPUhXYT1oO4DPkss1+T6CeCWoDTP1N5xmIW2tZRG9rF8dAu7KqLJbb1OA2jfzyhJ9bMweDbifv7g28CUG1ZTE1O+keoJcYqIoSpj+0f6fuUPJZarkpMV4b0AMppOeUaCllgvpS66F5qY3uxKU4Wm6/AqeYmYrrjrRyKvjuaU3QzANONC6gxped3mop++hAqA44fP86dd97Jvn37CAaDNDQ00NTUxDe/+U327dtHfn4+d955J9///vcxnvydTygU4lvf+ha///3v8fl8LF++nB/96EesWHH+K8K8Xi9ut5ur7J/AqJjTlV7GqSmuN13sDv/9uf9CycXG3JfZ0VImVD0fOHfQRchwuHGymzDh9EhkspswoWJ6hNdDjzE0NITLdfZLIxkvcpNFitzFQ4rcxUGK3MXjf3ORk794IoQQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaxsluQKapebmoqmWymzFhYi2tk92ECTf1ucrJbsKEU+5un+wmTLjI5sLJbkJa2Oy2yW7ChNOGhye7CRNK1+PnHSsjOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWMk92AVD772c8yODjIM888M27MunXrWLx4MT/+8Y/T3p4m3z4avDuJxIdxmguYk3cluZaSlLEt/gO0B47gi/QB4DYXMSN3bVJ853AdLf79eCPdRLUQl5R8Epe5KO15JLVTr6OJ40QI4cDNLJbgVvJTxnbrbTRwlCB+NDRsOJjKTEqVqUkxrdTjY5AoEVaxAaeSm6FsRrU3vkPribeIhH04XKVUz/sIztzKlLG9HQdpqX+dYKAPXY9jtRdQPu1yiiuWpoyvPfAUnc3bmD73RsqnXZbONJK0/3EPLY/tJNIfwFFdSPVXr8I1uzRlbOfLBzn+ry8nbVNMBi578esAaLE4jf+1hf5tDYQ6BzHaLeQumcq0z1+GpcCR7lQS2pvepaVhE5GIH4ezlOo5H8Y1znkK+LpoqnsF31Ab4dAg02ffSEXVpcnHa95KR/NWQsEBAGyOYqbWrCe/cFbaczmlKbCfBv+ekX7CVMAc9+XkmovHje8M1lHr20ow5sNmdDPLdQmFOVUAaHqcWt82ekKNBONejIoZj6WSma415Bgyd55a4rU0akdG+gkll9nqMtyqZ9z4Lq2ZuvgBQgSw4aTGsIhCtey0x1to1erw6QNEibDaeC1OJS+tOXwgi9wHSUfgGEcH3mJe/lXkWkpo9O5hZ/fTXFZ2JxaDbUx8f6iVUtss5uSVoipGGrw72dn9FJeWfoYc48ibM65HybOUUWKbyaH+1zKdEp16C8fZzxyW4iKfFmrZw9tcol+LWckZE2/ExDRmY8eJgkovHRxmJ2bdgkcZKd5xYuRSQDGVHGFXplMCoKd9HyeOPEfN/Ftw5k6hvWEzB7f9hmXr/hqzZWzHYDRbqay5Cpu9EEU10t99hOP7H8dssZN3RufY23kQ32AzZosrU+kA0P3GUep/uYkZX9uAc04pbU/u4uDfPsny//oLzHlj338ABpuZFQ//xegGZfSfWiiGv7aLqZ9ejb26kJgvRP3P3+DQPzzD0p9/Os3ZjOju2Ef90eeYMe8WnLmVtDVu4eDO37D8stTnSdMi5Fg9FJQs5MTR51Ie05LjYtqs67DaCtDR6WrbzaHdv2XpJX+F3Tl+oZkoHcFajg5tZl7uOnJNJTQG9rKz71kuK/pUyn5iINLBvoGXmelaQ6Glio7gcXb3v8AlhR/DafIQ12N4Iz1UO1fgNBUQ08IcGXqb3f3Pc0nhx9KeD0Cn1swxbQ9zDMtxKx6a48fYHX+TtcqHUvYTg1ovB+LvUqMupEAto1NrYl98M6uVa3Cc/MIbJ0auUkixOoUj8R0ZyUOmK8+h0bebSsd8KhzzcJg8zMtfj0E10uY/lDJ+UcH1THEuwmUuwmHKZ37+BnSgL9SciCm3z6HGvRpPTupvrunWzHHKmUaZUoVDcTGbpRgw0E5jyvh8pYgipRy74sKmOJiizMCBm0F6EzGlylSmK3PJJ7Mj0tO1NbxNSeVKSipXYHcWU7PgFlSDia6W1B+mXE81BSXzsTmLsdo9lE+7FLuzhKH+xqS4cGiI+kN/ZNbij6OohgxkMqrtyV2U3rCAkuvmY5/qYcbXr0a1mOh86cD4OykK5nz76C3PnnjI6LCw8Ae3U7huFrbKfFxzy6j56nr8x7sIdXkzkBG0NW6mtHIlJRXLsTuKmTHvZlSDmc62nSnjne5Kps++gaLSRShK6tffUzSX/MLZWO0F2OyFTJt5LQajGe9Qc8r4idbo30ulbR4Vtrk4TPnMc1+JQTHSNnwkZXyTfx8FlilMcyzFYcpnhms1LlMhzYH9AJhUCysKPkKpdQYOYx655hLmui/HG+0hGPNlJKcm7SgVajXl6nQcips5hhUYMNKmnUgZ36wdw6OUUmWYg0NxU2NYiEvJo1mrTcSUqdOoNszHo6T/i8cpaStymqbxgx/8gJqaGiwWC1OmTOEf//EfAThw4ABXXXUVVqsVj8fDF7/4Rfx+/7jHCgQCfOYzn8HhcFBaWsqDDz6YrmYn56DH8Ua6k4qRoih4cqYwGOk4r2PE9Rg6cUyGsd98JoOma/gYTCpGiqKQTzGD9J1zf13X6de7COAjl8J0NvWCaFoM31AbuQUzEtsURSW3oAbv4Lk7Ol3XGeitIxjowZ0/7bTtGsf2PkrF9CuwO1NPUaeLFo3jO95F7tIpiW2KqpC7dAq+w+O//+LBCNs++Wu2fuJXHPr7Zwg09o4bCxALhEEZKYDppmkxfN42cj01iW2KopLrqcE32DQhz6HrGt0d+4jHIrhyp5x7h/dJ0+N4o914LGf0E5YKBqOdKfcZjHYmxQMUWKYwGEkdDxDVI8BIAUw3TY/j0wfIP60YKYpCvlLMkJ66nxjS+5LiATxKCUPaufuVdErbdOW9997LQw89xI9+9CMuvfRSOjo6OHr0KIFAgGuvvZY1a9awY8cOuru7+fznP89Xv/pVHn744ZTH+uY3v8mmTZv44x//SFFREd/+9rfZvXs3ixcvHvf5w+Ew4XA4cd/rvfBvqZF4EB0d8xnTDRbVRiDaf17HODa4GYvBgScn/R+28xElPJITyUXXjIUA479GMT3K2zyHhoaCwiyWZPTb2LlEI8Oga2Omu8wWJ8FAz7j7xaJBtm38J3QtBopKzfybySucmXi8tX4TiqJSVrU2bW0fT3QoCJqeNBIDMOfZGGpJ/f6zVeYz66+vxT69kFggTOvjO9n7V79n+W8+i6XQOSZei8Ro+I+3KLxyNkZ7+jvPxHkyn3meHAyd5Tydj4Cvkz1bf46mxTAYzMxbegd2R/rfoxHtVD9hTdpuUW0EIoMp9wnHhzGrZ/QrBhthbThlfFyPcdz7DqXWmRhV84S0+2wiRFL3E0oOAT11PxEmNGYa00wOEYJpa+f5SEuR8/l8/OQnP+FnP/sZd955JwDV1dVceumlPPTQQ4RCIX77299it498eH/2s59x00038S//8i8UFye/Kf1+P7/5zW/4n//5H9avXw/AI488QkVFxVnb8MADD/Cd73wnDdmdvxNDO+gcPsbKotswKBf35U8DRlZxNXFi9NNNLfux6nbylcmbnpwIBqOFpZd9jXgswmBfHScOP0eOLZ9cTzW+oVbaGjez5NKvoSjKuQ/2AeCaW4Zr7uiFfte8Mnb+xcN0PLefqj9PLtRaLM7h7/0JdJjxtQ2ZbuqEs9oLWHbJXxGLhejtPMix/Y+zcNUXM1Lo0knT4+ztfwmAee51k9uYi1Baet4jR44QDocTRenMxxYtWpQocABr165F0zSOHTs2psjV19cTiURYtWpVYlt+fj6zZp191dS9997LPffck7jv9XqprLywa2BmgxUFhUg8+dtVWBvGYrCPs9eIBu8uTnh3sKLoVpzmD860ngnLSE6EkrZHCI/51nY6RVGwMfLt20kuAd1LI8cm9Rrc6UxmGygqkXDytHck7MNkGTuCOUVRVKz2AgAc7jKG/d201L1Brqcab38D0XCA7a8/MLqDrnHi8PO0NWxh5VV/m5ZcTjG5raAqRAYCSdsjA8NjRnfjUY0GHDVFBNsHkrZrsThHvvcc4S4fC//19oyM4uC08xQ58zz5Uy46uRCqakycS6e7Ap+3lbbGLcyc/9H3ddxzMaun+onkEctIP5F6cZDFYCNyxqgtHB/GcsboTtPj7B14mVDcx4qCmzMyigMwY07dT+ghLFhT7mMhh4h+Zr8SwjxOfKak5Zqc1Tq5SQFYLBZcLlfS7UKpigGXuYi+UEtim67r9IVayDWnXsINcMK7k/qhbSwvugW35YP1LVJVVJzk0k93Ypuu6/TTTS7jLw0+kw5oxNPQwvdGVY043eUM9tYltum6xmBf3YVdl9F1dG0kr6LypSy9/OssvexriZvZ4qKi+grmr/zcRKcwhmoy4JxZzODu0WuKuqYzuKcZ59zx33+n0+MagYYezPmjBeRUgQu2DbDgB7eNFNMMUVUjTlc5g31jz5Mzd+pZ9rxwuq6NTEOnmaoYcJmK6Iuc0U+EW8k1pb6Om2sqoS/cmrStL9xCrnk0/lSBG44NssJzM2Y1g+dJMeBU8ujXuxLbTl2Pdyup+wm34kmKB+jTO8/6k4NMSEuRmzFjBlarlY0bN455bM6cOezbt49AYPTb6ZYtW1BVNeXorLq6GpPJxLZt2xLbBgYGOH78eDqaPkaVcymt/oO0+Q/jj/ZzaGAjcS1KuWMuAPt7X+bY4OZE/AnvDmoH32W+52qsRhfheIBwPEBMiyRiIvEQ3kh34rpeIDqAN9JNOJ78jT1dpjCTdhpo1xsJ6F6Osps4MUqpAuCgvp06fXT1XoN+lD69i2HdT0D30qQfp5MmShntlKJ6BJ8+mLiuF8CHTx8kfMY3u3Qqn3YZnS3b6WrdxbCvi7qDT6PFohRXLgfg2N5HaTj6YiK+pe4NBnqOExzuY9jXReuJt+hu201R+RIATGY7dmdJ0k1RDZgtDmyOzIzOy29dRscLB+h85RDDTX3U/uQ1tFCUkuvmA3D0n1+k4T/eTsQ3/fe79O9sJNg+iK+2i6P//ALhLh8lNywATha47/wJ3/FOZt97A2g6kf4Akf4AWjQzX1rKqy6lo3UHnW27GPZ3U3voGbR4hJLyZSM57X+UhmMvJeI1LYbf247f246ux4mEvPi97QQDowtqGo69xGD/CULD/QR8nTQce4mh/gaKypZkJKcqx2JaA4dpGz4y0k8MvUlcj1FumwPA/oFXOeZ9JxE/1bGI3nAzDf49+KMD1Hq3MRTtZop94UjOepy9Ay/hjXSzMO8adLREX6LpmTlPU9XZtGn1tGsN+PUhjmg7iROjTJ0OwMHYVmrj+xLxU9RZ9OkdNMaPEtC91McP4NUHmKKOLgaL6mF8+gD+k9f1AroPnz5AWE/fdbu0TFfm5OTwN3/zN3zrW9/CbDazdu1aenp6OHToEJ/61Ke47777uPPOO7n//vvp6enhrrvu4o477hgzVQngcDj43Oc+xze/+U08Hg9FRUX83d/9HaqamV8/lNpnEdGC1A69Szg+jMtcwPKimxPTlcG4N+l3SM2+/ejE2dv7fNJxql2rmJG7BoDuYD0H+19NPLav78UxMelUolQS1cOc4DBhQjhxs4RLsZy8aBxiGOW0pOLEOMoewgyjYsCOk3mspEQZnf7toZ3DjC4BP8jIl5JpzKGaeWnPCaCwbBHRSICm46+c/DF4GfNW/gXmk9OV4eAgnHZtLR6PUHfwGSKhIVSDCau9kFmLP05h2aKMtPd8FF05m+hQkKaHtxAZGMZRXcj8B25NTFeGu70o6mhOMV+I2n97hcjAMEaHBeeMYhb/5OPYp458m470+ul7tx6A3X/530nPtfCHf0bu4vT/rKWo9OR5qn01cZ7mL08+T6e//yIhL7vf+ffE/dbGt2htfAt33jQWrfrLkZiIn2P7HyMS9mE05WB3lrJg+V+Qd9pq23Qqtc4Y6Sd82wnHA7hMhSz33JSYrgzGfZzeUeSZS1mUdw3HvVs57n0XuzGXpfk34DSNnKdQPEB3qAGAd3r+kPRcKzw347GcfU3CRChRpxDRQ9THD4z0E0ouSw3rTusnAiNTOiflqgUsYA118QPUafux4WSR4dLEb+QAevQ2DsW3J+4fiI8U/unqPKoNC9KSh6Lrun7usAunaRoPPPAADz30EO3t7ZSWlvKlL32Je++9lwMHDvC1r32Nd999F5vNxq233sq//du/4XCMTKmc+RdP/H4/X/7yl3nqqadwOp184xvf4Pnnn7+gv3ji9Xpxu91sqPgyxgwswc2UWEvruYMuMuEbVkx2Eyaccnf3uYMuNj/84Fxrnki2/dn3mYr3nP1nJBebmB7ljdiTDA0NnfNSVNqK3AeNFLmLhxS5i4QUuYvG/+YiJ3/xRAghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1jJPdgEzTBgbRFPNkN2PiqIbJbsGEs71zfLKbMOFig1WT3YQJ13CzabKbkB5XTpvsFky4mb/Moj4P0LUwNJ1frIzkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1jJO5MHWrVvH4sWL+fGPfzyRh510zZGjNEYPEtGDONR85lhW4jYUpoz1xweoi+zFq/UR0gPMMq9gqnluUkxdeC8novuSttkUF5fab0lbDmdq0Wpp0o8SIYSDXGapS3ErnpSxfn2Ieu0gPvoJMcxMZTFT1FlJMQ3aYXr0VgL4UDGQSwE16kLsiisT6QDQHDpMQ/gAES2I05DPbNsaco3jn6fa4G688V5Cmp9Z1lVU5cxPPl74CC3hIwTjfgAchlyqrUsoNFWmPZfTtXRso7ltM5GIH4e9hJnTP4TbWZEy1j/cxYnm1/H52wmFB5kx7XqmlF2SFLNl54OEwoNj9i0vWcns6pvSkcIYQ1s2M7TpTeI+H+bSMjw330LOlCkpYwMH9jPw+kZivb3ocQ1TQQHuK67AuWx5Ulykq4v+F54jeOIExDXMxcUUf+ZOjHl5mUiJoXc2M/TWaTl95BZyKlPnFOnspP/Vl4i0tRIbGMBz40dwX3Z5UkzwRD1Db71JuLWVuM9L8Wc+i33egkykktDk3UvD0E4i8QBOcyFzPFeSaykdN74zcJzagS0EY15sxlxm5V9GoW36aY/X0uLdjzfSRVQLcUnZp3FZitKaw4QWuWzUGW3gWGQHcy2rcRsKaYocZlfwNdbabsaiWsfEx4ljVZ0UG6s4Ftkx7nHtai7Lc65J3FcUJS3tT6VTa+a4vpc5yjJciocW/Th7tE1cot6AWckZEx8nhk2xU0wlx/U9KY85qPdQoczApeSjo1GnHWCPtok16vUYlPS/zToiJzga3MY821rcxkKaQofY5X+JS123pT5Pegyb6qTEXMXR4W0pj5mj2JlpXYFNHSnU7ZFa9vhf4xLXzTgMmek4u3oOUNvwIrOrP4zLWUFL+7vsPfQIa5Z+DbPZMSZei0exWvIo8syjtuHFlMdcsehL6LqWuB8Y7mbPoYcpLpifMn6i+ffuoe9Pz1J4621Ypkxh6O236fyPX1P5rb/B4HCOiVdtNvKu2oCpqAjFYGD4yGF6HnsUg8OBbdZsAKK9vbT//Gc4V6wk75prUS05RLo6UUyZ6eL8+/bQ99yzFN5yMqfNb9P5m19T+depc9KiEUz5HhwLFtH33B9THlOPRDCXluFcvpKu/344zRmM1eE/xtG+TcwrWE+upZRG7252dj7FZRV/jsVgGxM/EGpnX/fzzMy7lELbdDoCR9nd9SyXlH8ap7kAgLgWJS+njBLHTA71vpqRPD7Q05WRSGSym0Bj9DAVphmUm2bgUHOZa1mDQTHQHqtLGe82FDDLspxS0zTUs7y8KgoW1Zq4pSou6dKsH6NcmU6ZOh2H4ma2shwDRtr1hpTxbsXDDHUxJeqUcXNaYriCMnUaDsWNU8ljnrqSEMN46U9nKglNoYNUWGZRbpmJw5DHXNtaDBhpixxPGe82FjLLtpJSczWqYkgZU2SeQqGpErvBjd3gZoZ1OQbFyGCsO52pJGluf4fy4uWUFS/FYStidvVNGAwm2rt3p4x3OSuYMe06SgoXoqqpO3izyY7F7EzcevuPYc3JJ9dVlcZMRg299RauVatxrliJubiEgo/eimIy4du+PWW8tboG+4IFmIuLR0Zxl12OubSUUMPo+7X/pRexzZ6D58absJRXYCoowD5vfsoCk5ac3n4L18rTcrrlZE47UueUUzkFz4duwrF4CYox9XmyzZ5D/rXXY5+f2dHbKY3eXVQ651PhnI/D7GGeZwMGxUib72DK+CbvbgqsVUzLXYHD7GFG3lpcliKavXsTMeXOudTkrcGTk3qEmw4TXuQ0TeNb3/oW+fn5lJSUcP/99yceGxwc5POf/zyFhYW4XC6uuuoq9u0bnba7//77Wbx4Mf/xH//BtGnTyMnJOa/90kXT4/i0PjyGssQ2RVHIN5QxGO95X8cOaD42BR7j7cCT7A+9RVDzv9/mnhdNj+NjgHylOLFNURTylWIG9d4Je54YUQBMmCfsmOPR9DjeeC8eY/J58pjKJqwg6bpGR6SeuB4j15je6ZVTNC2Gz99Ofu7odI+iqOS5qxnytUzYc3T27KOsaGlGZhP0WIxwWyvWGTMS2xRVxTpjJqGmpnPvr+sEa48T7e4hZ/rI66JrGsNHj2AqKKTjoV/ReP99tP37TwgcPJC2PJLaNF5ONTMJNZ87pw8iTY/jDXfhsU5NbFMUBY91KoPhjpT7DIY6kuIBCqxVDIbb09rWc5nwsfwjjzzCPffcw7Zt23j33Xf57Gc/y9q1a7n66qu5/fbbsVqtvPjii7jdbn71q1+xfv16jh8/Tn5+PgB1dXU8+eSTPPXUUxgMI9+wz2e/M4XDYcLhcOK+1+u94Fwiehgdfcwoy6LkENCGLvh4p7gNBcw3rMWuuAjrQeoj+9gRfIlLbB/BqJje83HPR5TISE4k52QmhwAX/hqlous6x7U9uCnAoeROyDHPJqKH0NHHTEuaFSuB+Hs/TwC+eD/bvH9CI45BMbHEsSFjU5XR6DA6GmZT8rSk2exgeGhivpD09B8hFgtRWrRkQo53LvFAADRtzAjL4HAQ7R7/C4kWDNL0/e+ix2Ioqornlo9imzlyXTju96OHwwy+8Tp5111H/g03Ejx2lK7fPkLpX34Za3V1enMaHicnp4NoT+ZG/RMpEg+O9BNnTEtaDDYC0dSzM+F4IGV8ODactnaejwkvcgsXLuS+++4DYMaMGfzsZz9j48aNWK1Wtm/fTnd3NxaLBYAf/vCHPPPMMzzxxBN88YtfBEamKH/7299SWDiyYGDz5s3ntd+ZHnjgAb7zne9MdHoTotA4umjACbgNhbwdeILOWCMVphnj73iROKrvws8Qy9X1k92U982uulnjuoWYHqEr2sCBwFusdN6QsUKXbu1du/HkzcBiydwCofdCsViouPsbaOEwwbpa+v/0LCaPB2t1Deg6ALZ588i9/AoALOXlhJoa8W59J+1FTnywpaXIna60tJTu7m727duH3+/H40lewRcMBqmvr0/cnzp1aqLAAee935nuvfde7rnnnsR9r9dLZeWFrYozKxYUFCJ6KGl7WA9hUcYuZnivTIoZm+oiqE3MSOqsz4V5JCeSc4oQGjO6ey+Oarvo1dtZrl5FjjL24nQ6mJUcFBTCWjBpe0QPYk6x6ORCqIoBu2GkALiNBQzFemkKHWKe/dL3ddzzYTLZUFCJRJOnsiMRf8pFJxcqGBqkf7CehbM/8b6Pdb4MdjuoKnG/L2l73O/H4Bz/+pmiqpgKRhYvWMrLiXZ3Mfj6RqzVNYljmouLk/YxFRUnXbdLF4NtnJx8Z8/pg8xssI70E/HkUVg4PozFYE+5j8VgTx1vzEw/MJ4JL3ImU/J0m6IoaJqG3++ntLSUN998c8w+ubm5iX/b7ckv4PnudyaLxZIY+b1XqmLAqXroi3dQZBy5UKrrOv3xDqaYZr+vY58upkcZ1nyUGtP/jVNVDDjJo1/vokgZGVHquk6/3kWl8t5Hkbquc0zfTY/exjL1SqzK+++Ez5eqGHAZCuiPdVBsrkq0py/azpScuWff+YLpaGjnDpsAqmrE6Sijf+gEhZ6RPHRdY2DoBBWlq9738Tu6d2M22fHkz3zfxzpfitGIpbyCYF1tYkGFrmkE62pxX7L2vI+j6zp6LD56zMpKoj3J18mjPT0Z+flAUk7z3ntOHySqYsBlKaYv1EyxvQY4+ZkKNjPVtTjlPrk5pfQFm6lyL01s6ws2kWspSxmfKRn7CcHSpUvp7OzEaDRSVVWV9v0mSpVpLgfDm3GpHtyGApojR4jrMcqMIyf+QOhtchQbMyzLgJELtv6T1+t0NEL6MN54P0bFmFiKfiy8g0JjJVbFQVgfpi6yFwWFUtO0jOQ0RZnFYX0bLi0ft+KhWT9GnBilysjzH9S2koONGnVhIqdT1+s0NMIE8ekDGDBiU0a+qR7Td9GpN7NIvRQDRsL6yKjKiCkjPyGYmjOfg4G3cBkKTv6E4CBxYpSbRzrwA4FNWFQbM60rEjn544PASOEIa8N4Y30YFFNi5HY8uIMCYwVW1UGMKB2RevpjHSxzXJf2fE6ZUnYJh2ufwuUox+Uop7n9XeLxCKVFIx3JoeNPYDG7qKka+TmKpsUIDPec/HeccNiLz9+BwWDGZh2dDdF1jY7u3ZQWLRl3dWm6uC+/nJ5H/4ClohJL5RSG3n4LPRLBsWIlAN2//x1Gt5v8Gz4EwMDrG7FUVGDyFKDHYgwfPYJ/1y4KPnpr4pi5V1xJ1//7b3KmT8daXcPwsaMMHzlM2Ze+nJmcLrucnsdO5lQxhaHNb6FHIziWn8zp0d9hdLnJv34kJz0WI9LddfLfcWLeIcLtbahmS2LEqoXDRPtGr71G+/sJt7dhsNoyUryrXMs40PsSbnMxbksJjd7dxPUo5c55AOzveRGLwcGs/MsAmOpayvaOx2gY2kmhdeQnBEPhLuYVXJ04ZiQeJBTzET7529NAdAAYGQVajKlHiO9Xxorchg0bWLNmDTfffDM/+MEPmDlzJu3t7Tz//PPccsstLF++fEL3myglpmlE9BD1kb2E9SBONZ+l1g2JRQ4hLYCijq5KC+tBtgb/lLjfFD1EU/QQeWoxK2zXnYwZ5kDoLSJ6GLOSQ56hiFW21L9RS0tO6hSiWpgT+kHCeggnuSxRr8By8vlD+nDSSrswIbZpr4zmpB+jST9GLoUsN1wFQKs+MnW8S3sj6bnmKispU9JfvEvN04loIepCuwhrQVwGD8sc1ybO08jq1dNy0oZ51/dM4n5j+ACN4QPkGUtY6RzpiCJaiAPDbxHWhjEpZhyGfJY5rqPAVJ72fE4pLlxAJBbgRPNGwhE/Tnspi+d9BsvJ6cpQeAhFGV0kHY742L7v54n7ze1baG7fQq6rimULPpfY3j94glB4iLLi0W/dmeJYvIR4IMDAyy8T83mxlJVT8vkvYDw5tRcbHITT3n96JELv008RHxxEMZkwFRVR9IlP4lg8uljGvmABBR+9lcE3XqfvmacxFRZRfMed5EybfubTpyenRSdzeuW0nP5i/JxiXi9tP/m3xP2ht95k6K03yZleTdlffgWAcGsLHb/+RSKm/7lnR55r2XKK/iz9U8yljllEtGFqB94hHB/GZSlkefFHE9OVwZiP0z9TeTllLCq6geMDWzjevwW7KZelxR9O/EYOoHv4BAd7X07c39fzPADVuauZkZf8RwsmiqLrJ6/aToBUf/Hk5ptvJjc3l4cffhifz8ff/d3f8eSTT9LT00NJSQmXX345DzzwAJWVldx///0888wz7N27N+m459rvfHi9XtxuN1fZP4FRSf+y9kzRgqFzB11kDK7MTXVmSmxu1WQ3YcI13Dy511rSJjOz0Rk185dtk92ECRXTwrzW9H8ZGhrC5Tr7oqkJLXIfZFLkLh5S5C4OUuQuHv+bi9wH+i+eCCGEEO+HFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1jJOdgMyTtcBfbJbIc5C8wcmuwkTzrC3drKbMOFqAlMnuwlpof/IO9lNmHAtPdl1ruLhEPz0/GJlJCeEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC3jZDfgYtAcPUpj9BARPYhDzWeOeSVuQ0HKWL82SF1kL16tj5AeYJZ5OVNNc5NiTkQO0B1vJqANoWIk11DITPNS7Ko7E+kA0KLV0qQfJUIIB7nMUpfiVjzjxnfpLdRrBwgRwIqTGepCCpSyxOOHtG106I1J+3goYYnhinSlkFJLvJZG7chIXkous9VluNWz5KU1UxcfycuGkxrDIgrV0bzq4wfo1JoJMYyKikvJp0ZdeNZjTrQLef8BdMYaqYvsJaT7sSkuZpiXUmisSIrxa4PURnYzEO9CQ8ehullkuQKr6kh3OgA09+ygsftdIjE/Dmsxc8qvw20vTxnbNXiEhq4tDIf70dCwm/OZWrSasvyFSXH+UA+17RsZ8DejoeGwFLBo2u1YzZn5XLU8vY/GR3cR6R/GUV3A7L9ah3tOScrY9pcOc+hfXk3appoMrH/lq4n79Q9vpfP144R6fKhGA66ZRdR87hLcc1MfMx369mymb8cbxAI+cgrLKFl/C7bSqSlj+/e/y9ChnYR6OwGwFldQdNkNSfHe4/vp3/cOoa5W4qFhpn/mG1iLUp/3iSJF7hw6Yw0ci+xkrnk1bkMBTdEj7Aq9xlrbR7Ao1jHxcT2GVXVQbJzKscjOlMcc0LqoNM7CbShA1zVqo3vYFXqNS6wfxqiY0p0SnVozx/W9zFGW4VI8tOjH2aNt4hL1BsxKzpj4Qb2Xg9q7VCsLKVTK6NSb2KdtYZV6NQ4lNxHnoYS56srEfRVD2nM5XafWzDFtD3MMy3ErHprjx9gdf5O1yodS56X1ciD+LjXqQgrUMjq1JvbFN7NauSaRl01xMtuwDKviQCNO0zmOOeE5XeD7bzDezYHw29SYllBorKAz1sDe8JusVj+EU80DYFjzsSP4EuWmGVSbFmFUzPi1QVQlM+erc+AQx9pfZW7FDbjt5TT1bGPXid+xdvZXsJjsY+JNBivTii/FnuNBVQz0eGs51PwsZqOdAlf1SE7hfnbUPkK5ZzHVJVdgNFjwh3pQlcx0cZ2vH+fYL95mzt1X4p5TQvMTe9n9rWdY+9vPYM6zpdzHaDdzyW8/M+4xbRW5zP7aOqylbrRwjKYn9rD7W0+z9n/uxJyb+pgTaejoHrre/COlG27HWjqF/t1v0fTEr5nxF3+L0e4cEz/cUo979lJKyqtQDUZ6t79O0xO/ouaz38LkzAVAi0awlU/DPWsx7a88lvYcQKYrz6kxeoQK4wzKTTU41FzmmldjUAy0R+tSxrsNBcwyL6fUOA11nJd3Wc6GxPGchnzmW9YS0gN4tf50ppLQrB+jXJlOmTodh+JmtrIcA0ba9YaU8S36cTyUUKXOxq64qFYX4CSXFj35NVAxYFGsiZtJMWcinYQm7SgVajXlJ/OaY1iBASNt2omU8c3aMTxKKVWGOTgUNzWGhbiUPJq12kRMqVqFRy3BpjhwKG5mGZYQI4pPH8xIThf6/muKHsFjKGOaeT4ONZca8xJcaj4t0WOJmLrIHgoMFcw0L8Nl8GBTnRQZK1MWzbTk1LOVCs8Syj2LceQUMrfiQxhUE+39e1PG5zurKM6djSOnEJsln6mFq3BYixkMNI/m1PEGBa4aZpZtwGUrxWbJp8g9K2XRTIemx3dT8aF5lF8/D0eVhzn3XIUhx0jbi4fOup8l3550O13phtl4lk3BVubGMc3DrK9cRiwQwVffm85UEvp2biJvwWryFqwkp6CE0qtvQzWZGDi4PWV8xYc+Tf6StViLyrF4iim79mOg6wSaRz9PufOWU3TJtdinzsxIDiAjubPS9Dg+rY/ppvmJbYqikG8oZVDrmbDniekRgIwUBU2P42OAKmVOYpuiKOQrxQzqqT88g3ofU5XkN6VHKaVHb03aNkA3m+LPYMJMnlJEtbIAs2KZ+CRS0PQ4Pn2Aaero1PCpvIb0vpT7DOl9TFFnJW3zKCV0a22kGoRqepxWrR4jJpxK3oS2P5X38v4b0nrGTI97DGV0x1sA0HWdnngrVab57Aq9ijc+gFV1MN00nyLjlPQlc5KmxfENdzC9aG1im6Io5DumMRhoPcueI3Rdp9/fSCDcx0zH+sS2Hm8dVUVr2FX///AGO7Gac5letJai3Nlpy+UULRrHd7ybaZ9akdimqAr5S6cwdKhz3P3iwShvf/w/0TUd14wiaj5/CY5pqafBtWic1ucOYrSbcdYUTngOY54vHiPY1UrBqvWJbYqiYp8yk2B74/kdIxZB1+IYctI/6jybrC1y4XCYcDicuO/1ei/4GBE9jI6O+YxvuBbFSkC78OOlous6RyM7yFULE9NJ6RQlMpITyVNtZnIIkDqnCKEU8RYihBL3PZRSpFZgxc4wfuq1A+zV32KFuh5FSf+EQWS8vJQcAnrqvMKExkw5mskhQjBpW4/WxoH4u8SJYcHKUsO6jBTv9/L+C+uhMfFmJYeIFjx5zBBxYjREDzLDvJgZpmX0xdvYG36T5co15BvSe70nEh8eycmUfO3PYrITCI8/QonGQ7x16MdoWhxFUZhTcQMe5/SRY8YCxLUIDd3vMKNkHTNK19Pnq2dv4+Msr/kM+Y7U15AmLKehILqmj5mWNOfZCDSnnp2xVeYx91tX46wuIOoP0/TYbnbc9Rhr/uvT5BSOTgX2vHuCA999iXg4isVjZ+kPb8HsTv+IOx4MgK6NmZY02p0M93ef1zG6Nj2H0e7O6Kgtlawtcg888ADf+c53JrsZ53Qksg2/NsjKnOsmuynvS4k6OgpwkItDzeUd7XkG6CGf4kls2fuXrxSz2ngtET1Mm1bP/vg7rFKuzsg1uYmmowNQZKhIjPhchnwGtR5ao8fTXuTeK6NqYc2sLxKLR+j3N3Cs7RWs5lzynVWjOblmMrVoNQAuWwmDgRZae3elvci9F7nzSsmdVzp6f34p79z537T+6SA1f7EmsT1/cSWr/+OTRIaCtD13kP3feZFVP//YuNf5Pih6tm3Ee2wPVR/7P6jG9K8zOJusvSZ37733MjQ0lLi1tLRc8DHMigUFhYie/M0+rAexTEAHdyS8jZ54K8tzriFHzcy1AxPmkZxOG4VB6tHaKSOjmzPjw+PGA9gUByYsDOu+99/o82AeLy89hIXU33wt5BDRU70OyfEGxYhNcZKrFjDPuAoFZdzrfBPpvbz/LErOmPiIHsKsWpOO6VBzk2LsqpuQHpi4xo/DbLCN5BT1J20PRwNYjOOv7FQUBZslH5ethKqiNRTnzqGhe8tpx1Rx5CRP49lzCghFhyY+iTOY3VYUVSEyMJy0PTIwPOY623hUowHnjEKG2waTthusJmzlueTOLWXet65GMSi0vXD263wTwWC1g6ISCyR/fmMBX8pFJ6fr3fEGvds3MvW2L5FTWHbW2EzI2iJnsVhwuVxJtwulKgacqoe+eEdim67r9Mc7yVXf+7y4ruscCW+jO97M8pxrsKlnf9NMJFUx4CSPfr0rqT39ehe5Supl6bmKh349eYqiX+/EPU48QEgfJko4Y4sZVMWAU0md13g/jXArnqR4gD698zx+HqCjEX+/TT6n9/L+c6uF9MWTrwP1xTsS8apiwKUWjJnuHNa85Cjp/6KlqgactlL6/I2JbSPX2RrItVeMv+MZdHQ0LZ44pstWRiCcfO11ONxPjin9Px9QTQacM4vo3z36RVrXdPp3t+Ced34jYz2u4T/Rd+6iqI9cn0s31WDEWlyRtGhE1zUCzbVYy6rG3a93++v0vPsqU2/9ItaSyrS383xkbZGbKFWmObTFammL1uPXBjkS2Upcj1FmqgHgQHgztZHdiXhNj+ON9+ON96OjEdKH8cb7GT6tUzkS2UZH7AQLLJdhxERYCxLWgsT1WEZymqLMol0/QbvWQED3clTfSZwYpco0AA5qW6nT9ifiK5WZ9NFBk3aUgO6lXjuIlwEqlZHXIKZHqdX2MqT3EtQD9Otd7NM2Y8OBh8xNf01VZ9Om1dOuNeDXhziijeRVpo5cuzkY20ptfN/o66DOok/voDF+Mq/4Abz6AFPUGcDIz0Fq4/sY1Eby8ur9HIptI0yQYjX9izTgwt9/U01z6Iu30Rg9REAbSvxms9I067RjzqMz3khr9DjDmpfm6FF64q1JMWnNqXA1bX27aevfhz/Uw5HWF4hrUcryF43k1PQMte0bE/EnujbT5zvBcHgAf6iHxu536eg/QGn+gtFjFq2hc/AQrX27GQ7309yzg56h41QWLM9ITlNvX0rbcwdpf+kw/qZ+jvzodeKhKGXXjUwJH/ynl6l9aEsivv6RbfTtaGK4fQjv8W4O/NPLhLq8lH9oHjCyKKX2oS0MHu4g2OnFe6yLQ//yKuEeP8VXzMhITp7lVzCwfyuDB3cQ7uui49Un0KIR8uaP/Eyo9YXf0fXWc4n4nm0b6d7yIuXXfQyTO59owEs04CUeGV0bEQsGCHa3Ee4b+SIW6e8m2N1GNDAxaxxSydprchOlxDiNiB6mPrqXcCSIU81nac76xAglpAVQVCURH9aDbA2Nnvim6GGaoofJU4tZYb0WgNbYcQB2hl5Jeq555ksoP9l5pVOJOoWoFuaEfpCwHsJJLkvUKxJTYCF9GEUZzSlXKWC+uoZ67QB1+gFsOFikrk38lkxBwacP0a43EiOKhRw8SgnTlQUZ++3Vqbwieoj6+AHChHAquSw1rBvNiwAnL9+M5KUWsIA11MUPUKftx4aTRYZLT/vtn8Kw7mO/toUIYUyYcSselhvW41Ay8wPjC33/5RqKWGC5jLrIXmoje7ApLhZb1iUtaio2TmGuvoqG6EGORnZgV10sslxBniEz105L8uYRiQ1T37GJcMyP01rM0umfxHJyMUoo4kVhNKe4FuVIy4uEol5U1YjdUsCCqTdTkjdvNKfc2cyNf4iGri0cbX0Zu8XDomm3k+fIzJeRkqtmEhkKUv/wVsL9wzirC1j6LzcnRmahbh+cdp5i/hCHH9xIuH8Yk8OCa2YRK372ZziqTs4iGBSGWwbYf9/zRIZCmFw5uGcVs/zfbxt3BeZEc89eQmzYT/eWl4gNe8kpLGfqbV9MTFdGvQNJ/cTAvnfQ43Fann0k6TiFa66haO3ImgNf/SHaX/pD4rHW5/57TMxEU3Rd188d9sHzs5/9jKeffpqNGzeeO5iR1ZVut5urbB/HmOHfb6WTFgqfO+gic3qnnS0Uc/a85xJmfPAWdEwE/UfpG1VMlvbns+tcxcMhjv702wwNDZ3zUtRFO13Z29tLfX39ZDdDCCHEB9hFW+Tuv/9+GhsbJ7sZQgghPsAu2iInhBBCnIsUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrCVFTgghRNaSIieEECJrSZETQgiRtaTICSGEyFpS5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKxlnOwGZJo2HERTYpPdDHEWujbZLZh4uqZPdhMmnHKodrKbkBZDDy2f7CZMuP0P/nyymzChvD6NvJ+eX6yM5IQQQmQtKXJCCCGylhQ5IYQQWUuKnBBCiKwlRU4IIUTWkiInhBAia0mRE0IIkbWkyAkhhMhaUuSEEEJkLSlyQgghspYUOSGEEFlLipwQQoisJUVOCCFE1pIiJ4QQImtJkRNCCJG1pMgJIYTIWlLkhBBCZC0pckIIIbKWFDkhhBBZS4qcEEKIrGWc7AZcDFr0Opo4ToQQDtzMYgluJX/c+C69lXoOESKAFQczWECBUpp4PKbHqOMAPbQTJYwVO5XUUKFUZyIdYGJz0nSNeg7SSydBAhgxkU8RM1iARbFmKiUgO/Nq0Wpp0o+ezCmXWepS3Ipn3PguvYV67cDJnJzMUBdSoJQlHu/WW2nV6vAxQJQIq9RrcCp5mUgloSVeS6N2ZCQnJZfZ6jLc6lly0pqpi4/kZMNJjWERhepoTq9G/5ByvxnqIqoMcya8/SnbeHwLHUfeJBr0YcsrZeqyW3AUTEkZ2123ld6GXQQHOwGw51dQsej6pPho0EfL3ucZ6jxOPBLEWTSdqctuJsdVmJF8AH7+X4P88OeDdPbEWTTXzE/+sZCVS3JSxl710VY2vRsas/369Tae+5+Rc/XnX+vit4/5kh6/Zp2NF39fNma/iXJBI7l169ahKAqKorB37940NemD1YZOvYXj7Gc6c1nJBpzksoe3iehjTybAoN7LQbZRRhWr2EARZezjHfz6UCKmln300ck8VrCGa6lkBsfYS4/envZ80pGTRhwfg0xnDqvYwCLWMIyPvbyTkXxOyca8OrVmjut7ma7MY6V6DU4llz3aprPnpL1LmTKdVeq1FCnl7NO24NcHEzFxPUauUkiNsjBDWSTr1Jo5pu1humE+q4zX4iSX3fE3x89J6+VA/F3K1emsMl5LoVrOvvjmpJwuN34k6TbXsBKAIrUyEynR17SX5t3PUj7/auZf/3VsuWUce+MhoiFfynhfVz2eqYuZveFLzL3mLsx2N8fe+DWR4ZH3nq7rHH/rYcL+PmZc/lnmXX83ZnseR1//FfFYOCM5PfpHH9+4v5e//0Y+O1+uZOFcC9d/op3u3ljK+Cd+U0rbvqrEbf+blRgMcNtNjqS4a6+0JcX97hfFac3jgqcrv/CFL9DR0cH8+fNpbGxMFJwzb1u3bk3sEwwGue+++5g5cyYWi4WCggJuv/12Dh06lHTs4eFh7r33Xqqrq8nJyaGwsJArrriCP/7xj4mYp556iu3bt7+PlC9MM8cpZxplShUOxcVslmLAQDuNKeNbqMNDMVXKLOyKi2plPk7yaKE+ETNIH6VMJV8pwqrYqVCm48DNEP0XZU5GxcRS5XKKlUrsihO34mEWS/AxQEgfzkhO2ZpXs36McmU6Zep0HIqb2cpyDBhp1xtS56Qfx0MJVerskZzUBTjJpUWvS8SUqlVMV+eRr5RkJIczNWlHqVCrKT+Z0xzDCgwYadNOpIxv1o7hUUqpMszBobipMSzEpeTRrNUmYiyKNenWo7WRrxRhUxwpjznROo9uorB6FYXVK7G6S6haeSuq0URP/Y6U8dVrP0XxzLXY88qxuouYtvLP0HUdb+dITiFfL4G+JqauuBWHZwpWVxFVKz6KFo/S17g3Izn9+FeDfP5Tbv784y7mzjLzix8UYrMq/NfvUxfu/DwDJUXGxO21TUFsVoXbzyhyFrOSFJeXa0hrHhdc5Gw2GyUlJRiNozOdr732Gh0dHUm3ZcuWARAOh9mwYQP/+Z//yfe//32OHz/OCy+8QCwWY9WqVUnF8Etf+hJPPfUUP/3pTzl69CgvvfQSt912G319fYmY/Px8CgszM1zXdA0fg+RTlNimKAr5FDNIX8p9Bukjn+RvJh6KGTotPhcPvXQQ0oPouk6/3s0wfjyk9xsNpC+nM8WIAmDENAGtPrdszEvT4/gYIF8ZbaOiKOQrxQzqvSn3GdT7kuIBPEopQ+PEZ5qmx/HpqXMa0lO/7kMpcyphSEsdH9ZD9OrtlKnTJ67hZ6H9/+3dbWxT1x3H8e+9Tmw3xLFDeAh5ANJCeAolUCiUNRREESurxNqxFcSLVeu6aZUmDV4MhU50AgbSJFb6ppoY6kQlYHTrpLEVdS3QbgNC6QqEhABpCEkDDiEkaZwnP8T37IVJjIkTkmI7y+X/kfzm6tzr83OM/+ece64IdtPRfANnZn7vMU3TScucSvvt2kFew49SQSy2FACUEZot6Zbw76ym6eiWJNobow9wYsnvV3xxwcfyovCyvK5rLC9KoeSL6DPue71z0MNLqx2MSoksM/8q6SKz4Boznq7ltU23aGoOxrTv94rJPbmMjAwyM6OPCnfv3k1JSQnnzp1jzpw5AEyaNIn333+fhQsX8sorr1BeXo6maRw+fJi33nqLVatWATB58uTeYjkcAvhQKKxErkFbsdGBJ+o5frxYsd3T3o6f8BdjGoVc4iwn+AANDdCYwROka/Ev3vHKdLegClJFGZnkkqQlpsiZMVcAfz+Z7PfJ1Pcz6C9Tovn7y6TZ6VDRM/nwYtX6fgZ+uqK2rzeuYSGZcVpiliq7fR2gDJLskTOWZLsDr+fWoK5Rd/4DrI84cWZOBcCeNg5riovrpUeY/OQadIuVm1f+jb+zFX9X9M8plm43BwkGYfzYyFnW+LEWrlT573v+mXNeyi/7+cPvxkUcX7kshRdWpZI3MYmrNQF+tbOJ76x3c/IfOVgsWkwz9Ij7xpMDBw6wYsWK3gLXQ9d1NmzYwPr16yktLaWwsJDMzEyOHDnCiy++iMPheKD39fl8+HzhtWuPJ/5fjMGqo4pWmpjDYuyk8DW3ucI5bMpOhhb/2Vw8GcqgjNDsfDrzhrk3sWPWXGZ0w6hmgj4JixbfZbBYcV88TlPteWYs/xm6JTR40nULU5e8zLXT73H2L1tA03FmTsU5YTqghrfDg/DOAQ+zZ1j7bFJZ+93w7/rsGTYen2lj6qJaPj3VxfKilLj0JSaPECxevJjU1NSIV4/KykpmzIi+u6nneGVlJQB79uzh1KlTZGRksGDBAjZs2MDJkye/UZ927tyJ0+nsfeXmDn1Ul4wNDa3PKNiPr89ItEdohOm7p314dB2aDZSTzxzGalk4NBe52hTGk8NXVA65j0MVj0w9egqBl07mUpSwWRyYM1cy1n4y9e1jj2gz0YE+g0Sz9pdJebERfceqDXufTSmhz6Bv+xbjFp20kZ2gpUqAJNso0HS6ve0RxwPeNpLtaQOeW3/pU+orjjNt2U9ISY/cYThqdA4FqzYyb8025r6whWnLXqXb34Ettf9dqLEyZrQFiwUaGiOXEhsag4wfN/DcqKPT4NDf2vnRuoGzAzw6KZkxo3WqrgUeqL8DiUmRO3ToEOfPn4943U2pwY08lixZQnV1NceOHWPNmjVcvHiRoqIitm3bNuQ+FRcX09ra2vuqq6sb8jV0TceBi2bCSw5KKZq5hYvoXzQXGRHtAZppwHmnvcJARRmJaWhRj8daPDJBuBB00s48lmDVbPdeJq7MmEvXLDhIp1k19B4L3cNtwKWNiXqOS8ugWd2TSd3E2U/7RNM1Cw4teqb+HotwahkR7QGa1M2ojxzcMKpxaOkJfSRCtyQxanQ2rQ3hjTBKGXhuVpE6ZlK/59VXfIK7/CjTlr1Kakb/g/Ak6yMk21PxehrpaL5Oes6smPY/GqtV44nHbRw/EV4SNgzF8ROdPPXEwAOmP/+9HZ9fsf5791+Nu+7upqnFYML4+C0qxqTI5ebmMmXKlIhXj/z8fC5duhT1vJ7j+fnhG7bJyckUFRWxadMmPvroI7Zu3cq2bdvw+++/Dnw3m81GWlpaxOubmEg+bq7hVjV0KA+XOUuQbiYwGYBydYYqVdbbPpcpNHGTWlVJh/JwVV3EQwu5hJ6BS9KScTGGLymjWd2iS3XgVjXUU8s4sr9RH4c7k6EMLlCChxYKeBKFwqe8+JQXQxkJyWTWXBO1abhVNW7jWiiT+m8ok5YXymScpsq4EM6k5dNEPbXG5VAmozyUSQv/mwwoH22qhQ5C29U7VBttqgWfin6PK9Ym6dO5YVzFbVyjXbVyyQhl6tkoUt59mi+Dpb3tJ+rTaFL11ATvZAqW4VEtTNSnRly3WwVoUHVkJ/B50x6Z05+hseozGqs/p6u1gZrP/4rR7WfsowsAuHrqIHXnj/S2d1cc5/qFD8lb+AOso9Lxd3nwd3kIBsIrC81fleJpqMLb3kTL9XIuf7KH9JwCnBOmJSTTL37qYu9+D/ve83Cp0s9rmxrp6FS8vDZUvH748wY2/6bvhqY/HvCw+tujyBgduVzc3mHwy623Of2Fl5q6AMf+08kLL9czJS+ZlUvjs1QJCbgnt3btWl5//XVKS0sj7ssZhsGbb77JzJkz+9yvu9vMmTPp7u7G6/VitVrj3d0+MrVcAspHNRX48OLAyVyexnbnRriXzjubR0Jc2hgK1EKuUk4V5aSQyhwWk6o5e9vMZhFVlHGRMwTwY2cUj1FANolZYol1Jh9d3KYegM84GvFe81gSseNRcg0xkz6RgOGjWpXjU14cuJirPxPOpDrRtHsy6U9x1SijSpWFMunfIlVz9bZpVG4qVPgxnHJVAgrytFk8phUkJJNfebkaLAv9nTQX8yxL7/o7dUTcdnLpY5jNU1QFy6gyLpCCgzmWpyMyAdxUtb3XT7SMSYV0e9u5ceGfBLxtpKRnMW3Zj0l+JFQQ/J0tEX+nW1+WoIwgVSfejbhOVsEKch5fGTqny8NXZw8T8LaTbHcwJm8+WQXPJizTS6sd3G4K8uvfNnOzsZvCWTaOHMhi/NhQ2ai7EUC/Z5p0pcrPiTNePvxT34e7LTpcqPDx7nttfO0JkjU+iRXPpLB102hstvhsOgHQ1GDXEgk9iF1YWMju3bsBqKmpIS8vj6NHjzJrVuQU2uVyYbfb8Xq9LF26FLfbza5du1i4cCENDQ3s2LGDjz/+mKNHj7Jo0aLe669bt4758+eTkZFBRUUFGzduJDs7m2PHjvVeu+d9z507R2Fh4aD67vF4cDqdLGV1Qu8VCQGAPjI2QQyFpsfvh2k4tX5//nB3IeZKdv1+uLsQU542g/T8alpbW++7SheTmdyzz/YdXRw8eJC1a9dit9s5fvw4O3bsYPPmzdTW1uJwOFi2bBmnT5+moCA8cly5ciX79u1j8+bNdHZ2kpWVxfPPP8+WLVti0U0hhBAPmQcqcpMnTx7UppKUlBS2b9/O9u3bB2xXXFxMcXHxg3RJCCGE6DXkjSdvv/02qamplJWV3b9xHDz33HN9lkaFEEKIaIY0k9u/fz9dXaEdWBMnJv7mLsDevXuHvQ9CCCFGhiEVuezsxGxx/3/vgxBCiJFB/tNUIYQQpiVFTgghhGlJkRNCCGFaUuSEEEKYlhQ5IYQQpiVFTgghhGlJkRNCCGFaUuSEEEKYlhQ5IYQQpiVFTgghhGlJkRNCCGFaUuSEEEKYlhQ5IYQQpiVFTgghhGlJkRNCCGFaQ/r/5EYypRQA3QRADXNnxMNHGcPdg5jTlDbcXYiLYMA73F2IOU+bub5/nvZQnp7f9YFoajCtTOD69evk5uYOdzeEEELESF1dHTk5OQO2eWiKnGEYuN1uHA4HmhbfEajH4yE3N5e6ujrS0tLi+l6JIplGBsk0MpgxEyQul1KKtrY2srKy0PWB77o9NMuVuq7ft+LHWlpamqm+wCCZRgrJNDKYMRMkJpfT6RxUO9l4IoQQwrSkyAkhhDAtKXJxYLPZeOONN7DZbMPdlZiRTCODZBoZzJgJ/j9zPTQbT4QQQjx8ZCYnhBDCtKTICSGEMC0pckIIIUxLipwQQgjTkiInhBDCtKTICSGEMC0pckIIIUxLipwQQgjT+h8ErfO6+71U3wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 480x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "it s too cold here .\n"
     ]
    }
   ],
   "source": [
    "translator = Translator(model.cpu(), src_tokenizer, trg_tokenizer)\n",
    "result = translator(u'hace mucho frio aqui .')\n",
    "print(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "8570b29725840556",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:48:11.842644Z",
     "iopub.status.busy": "2025-01-27T15:48:11.842443Z",
     "iopub.status.idle": "2025-01-27T15:48:12.316240Z",
     "shell.execute_reply": "2025-01-27T15:48:12.315649Z",
     "shell.execute_reply.started": "2025-01-27T15:48:11.842624Z"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = Sequence2Sequence(src_vocab_size=len(src_word2idx), trg_vocab_size=len(trg_word2idx),encoder_num_layers=2,decoder_num_layers=2)\n",
    "model.load_state_dict(torch.load(\"checkpoints/seq2seq_layer_2.ckpt\",weights_only=True, map_location=device))\n",
    "\n",
    "\n",
    "class Translator:\n",
    "    def __init__(self, model, src_tokenizer, trg_tokenizer):\n",
    "        self.model = model\n",
    "        self.model.eval()  # 切换到验证模式\n",
    "        self.src_tokenizer = src_tokenizer\n",
    "        self.trg_tokenizer = trg_tokenizer\n",
    "\n",
    "    def __call__(self, sentence):\n",
    "        # 预处理句子，标点符号处理等\n",
    "        sentence = preprocess_sentence(sentence)\n",
    "        encoder_inputs, attn_mask = self.src_tokenizer.encode(\n",
    "            [sentence.split()],\n",
    "            padding_first=True,\n",
    "            add_bos=True,\n",
    "            add_eos=True,\n",
    "            return_mask=True,\n",
    "        )  # 对输入进行编码，并返回encoder_inputs和encoder_padding_mask\n",
    "        # 转换成tensor\n",
    "        encoder_inputs = torch.Tensor(encoder_inputs).to(dtype=torch.int64)\n",
    "        # 由输入得到预测结果，\n",
    "        # 注意，这里输入的样本只有一个，符合Seq2Seq模型的生成方法要求batch_size=1\n",
    "        preds, scores = self.model.infer(encoder_inputs=encoder_inputs, attn_mask=attn_mask)\n",
    "\n",
    "        # 通过tokenizer转换成文字\n",
    "        # trg_tokenizer.decode方法要求输入的是字符串列表，所以需要[preds]\n",
    "        # 方法返回字符串列表，所以需要[0]\n",
    "        trg_sentence = self.trg_tokenizer.decode([preds], split=True, remove_eos=True)[0]\n",
    "        \n",
    "        # 为了移除句子末尾的特殊标记\n",
    "        return \" \".join(trg_sentence[:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "18b2c2345791f9d5",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-27T15:48:12.317107Z",
     "iopub.status.busy": "2025-01-27T15:48:12.316894Z",
     "iopub.status.idle": "2025-01-27T15:56:48.097633Z",
     "shell.execute_reply": "2025-01-27T15:56:48.097007Z",
     "shell.execute_reply.started": "2025-01-27T15:48:12.317085Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 2-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 3-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n",
      "/usr/local/lib/python3.10/site-packages/nltk/translate/bleu_score.py:577: UserWarning: \n",
      "The hypothesis contains 0 counts of 4-gram overlaps.\n",
      "Therefore the BLEU score evaluates to 0, independently of\n",
      "how many N-gram overlaps of lower order it contains.\n",
      "Consider using lower n-gram order or use SmoothingFunction()\n",
      "  warnings.warn(_msg)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time used: 515.6192333698273\n"
     ]
    }
   ],
   "source": [
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "# from torchmetrics.text.bleu import BLEUScore\n",
    "\n",
    "# BLEU 分数的计算原理\n",
    "# n-gram 精度：\n",
    "# BLEU 基于 n-gram（连续的 n 个单词）的匹配程度。\n",
    "# 例如，1-gram 是单个单词，2-gram 是两个连续的单词，依此类推。\n",
    "# 计算候选翻译中每个 n-gram 在参考翻译中出现的次数，并除以候选翻译中该 n-gram 的总数。\n",
    "\n",
    "def evaluate_bleu_on_test_set(test_data, translator):\n",
    "    # 在测试集上计算平均 BLEU 分数。\n",
    "    total_bleu = 0\n",
    "    num_samples = len(test_data)\n",
    "    for src_sentence, ref_translation in test_data:\n",
    "        # 使用翻译器生成翻译结果\n",
    "        candidate_translation = translator(src_sentence)\n",
    "\n",
    "        # 计算 BLEU 分数\n",
    "        bleu_score = sentence_bleu(\n",
    "            [ref_translation.split()], candidate_translation.split(),\n",
    "            weights=(1, 0, 0, 0)\n",
    "        )\n",
    "        total_bleu += bleu_score\n",
    "    avg_bleu = total_bleu / num_samples\n",
    "    return avg_bleu\n",
    "\n",
    "translator=Translator(model, src_tokenizer, trg_tokenizer)\n",
    "\n",
    "start_time = time.time()\n",
    "avg_bleu=evaluate_bleu_on_test_set(test_ds, translator)\n",
    "end_time = time.time()\n",
    "print(\"Time used:\", end_time - start_time)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "460dd740-6aec-4398-a6c6-71ca45ac206a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-27T15:56:48.098625Z",
     "iopub.status.busy": "2025-01-27T15:56:48.098297Z",
     "iopub.status.idle": "2025-01-27T15:56:48.102039Z",
     "shell.execute_reply": "2025-01-27T15:56:48.101428Z",
     "shell.execute_reply.started": "2025-01-27T15:56:48.098602Z"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "avg_bleu:0.5271573672634846\n"
     ]
    }
   ],
   "source": [
    "print(f\"avg_bleu:{avg_bleu}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
